From 9951bf4c44c6427f749b4d10dc775c51329296b2 Mon Sep 17 00:00:00 2001 From: Abhijit Gangurde Date: Thu, 13 Aug 2026 16:19:54 +0530 Subject: [PATCH 001/153] RDMA/ionic: Add RCQ userspace support Expose the Reorder Completion Queue (RCQ) capability to userspace via ucontext response using a comp_mask, and allow userspace to specify ionic specific QP flags during QP creation. Co-developed-by: Allen Hubbe Signed-off-by: Allen Hubbe Signed-off-by: Abhijit Gangurde Link: https://patch.msgid.link/20260813104954.319065-3-abhijit.gangurde@amd.com Reviewed-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- .../infiniband/hw/ionic/ionic_controlpath.c | 20 ++++++++++++++++--- drivers/infiniband/hw/ionic/ionic_fw.h | 2 ++ drivers/infiniband/hw/ionic/ionic_lif_cfg.c | 1 + drivers/infiniband/hw/ionic/ionic_lif_cfg.h | 1 + include/uapi/rdma/ionic-abi.h | 9 ++++++++- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index a70ef59a8064..37f71fb43811 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -418,6 +418,8 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) resp.udma_count = dev->lif_cfg.udma_count; resp.expdb_mask = dev->lif_cfg.expdb_mask; + resp.rcq_sign_bit = dev->lif_cfg.rcq_sign_bit; + resp.comp_mask |= IONIC_CTX_CMASK_IONIC_FLAGS; if (dev->lif_cfg.sq_expdb) resp.expdb_qtypes |= IONIC_EXPDB_SQ; @@ -1388,7 +1390,8 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, struct ionic_qp *qp, struct ionic_tbl_buf *sq_buf, struct ionic_tbl_buf *rq_buf, - struct ib_qp_init_attr *attr) + struct ib_qp_init_attr *attr, + u32 ionic_flags) { const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject); const u32 flags = to_ionic_qp_flags(0, 0, @@ -1404,7 +1407,12 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, .len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN), .cmd.create_qp = { .pd_id = cpu_to_le32(pd->pdid), - .priv_flags = cpu_to_be32(flags), + /* User-supplied ionic_flags are passed through to + * firmware, which validates and rejects any + * unsupported or unauthorized bits. + */ + .priv_flags = cpu_to_be32(flags | + (ionic_flags & IONIC_QP_USER_FLAGS_MASK)), .type_state = to_ionic_qp_type(attr->qp_type), .dbid_flags = cpu_to_le16(dbid), .id_ver = cpu_to_le32(qp->qpid), @@ -2221,6 +2229,12 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, rc = ib_copy_validate_udata_in(udata, req, rsvd); if (rc) return rc; + + if (req.ionic_flags & ~IONIC_QP_USER_FLAGS_MASK) + return -EINVAL; + + if (req.rsvd_pad) + return -EINVAL; } else { req.sq_spec = IONIC_SPEC_HIGH; req.rq_spec = IONIC_SPEC_HIGH; @@ -2301,7 +2315,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, rc = ionic_create_qp_cmd(dev, pd, to_ionic_vcq_cq(attr->send_cq, qp->udma_idx), to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx), - qp, &sq_buf, &rq_buf, attr); + qp, &sq_buf, &rq_buf, attr, req.ionic_flags); if (rc) goto err_cmd; diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h index ee23062a1762..d0a5bce85ed8 100644 --- a/drivers/infiniband/hw/ionic/ionic_fw.h +++ b/drivers/infiniband/hw/ionic/ionic_fw.h @@ -105,6 +105,8 @@ enum ionic_qp_flags { IONIC_QPF_SQ_CMB = BIT(13), IONIC_QPF_RQ_CMB = BIT(14), IONIC_QPF_PRIVILEGED = BIT(15), + + IONIC_QP_USER_FLAGS_MASK = GENMASK(31, 16), }; static inline int from_ionic_qp_flags(int flags) diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c index 8e48c00d4959..1e37bd09490f 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c @@ -86,6 +86,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) cfg->udma_count = 2; cfg->max_stride = ident->rdma.max_stride; + cfg->rcq_sign_bit = ident->rdma.rcq_sign_bit; cfg->expdb_mask = ionic_get_expdb(lif); cfg->sq_expdb = diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h index 2b29e646c193..d7835ac27896 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h @@ -57,6 +57,7 @@ struct ionic_lif_cfg { bool sq_expdb; bool rq_expdb; u8 expdb_mask; + u8 rcq_sign_bit; }; void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg); diff --git a/include/uapi/rdma/ionic-abi.h b/include/uapi/rdma/ionic-abi.h index 2c70ac149c4f..3d8d56922a93 100644 --- a/include/uapi/rdma/ionic-abi.h +++ b/include/uapi/rdma/ionic-abi.h @@ -22,6 +22,10 @@ #define IONIC_CMB_WC 8 #define IONIC_CMB_UC 16 +enum ionic_ctx_resp_comp_mask { + IONIC_CTX_CMASK_IONIC_FLAGS = 1 << 0, +}; + struct ionic_ctx_req { __u32 rsvd[2]; }; @@ -46,8 +50,9 @@ struct ionic_ctx_resp { __u8 udma_count; __u8 expdb_mask; __u8 expdb_qtypes; + __u8 rcq_sign_bit; - __u8 rsvd2[3]; + __u16 comp_mask; __aligned_u64 phc_offset; }; @@ -85,6 +90,8 @@ struct ionic_qp_req { __u8 rq_cmb; __u8 udma_mask; __u8 rsvd[3]; + __u32 ionic_flags; + __u32 rsvd_pad; }; struct ionic_qp_resp { From 2f02eee8d632f768824c269cdf05c324e56e4d70 Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Mon, 31 Aug 2026 14:21:07 +0900 Subject: [PATCH 002/153] RDMA/mana_ib: Remove redundant NULL check before netdev_put() netdev_put() does nothing if dev is NULL, so the check before the call is redundant. So remove it. No functional change. This is the result of running the Coccinelle script from scripts/coccinelle/free/ifnulldev_put.cocci after commit f83b8a58695c ("coccinelle: ifnulldev_put: update outdated helper names"). Signed-off-by: Sang-Heon Jeon Link: https://patch.msgid.link/20260831052108.903019-1-ekffu200098@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/device.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index a8d19586a78a..62c7803ca6c6 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -103,8 +103,7 @@ static int mana_ib_netdev_event(struct notifier_block *this, ib_device_set_netdev(&dev->ib_dev, ndev, i + 1); /* mana_get_primary_netdev() returns ndev with refcount held */ - if (ndev) - netdev_put(ndev, &dev->dev_tracker); + netdev_put(ndev, &dev->dev_tracker); return NOTIFY_OK; default: From 71fc2d52047e79ecc7d27c68a503ff5af4542392 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Thu, 27 Aug 2026 07:37:34 -0700 Subject: [PATCH 003/153] RDMA/mana_ib: Extend RC QP udata request and response Extend the RC QP creation UAPI to support optional fixed-size WQEs and an MMQ. Add a compatibility mask to the request, return the MMQ ID in the response, and advertise the extended ABI through ucontext capabilities. Use compatibility-aware udata validation so existing userspace remains supported. Rename the legacy FMR queue definitions to MMQ and add the corresponding hardware creation flags. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/20260827143736.2013093-2-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/main.c | 1 + drivers/infiniband/hw/mana/mana_ib.h | 6 ++++-- drivers/infiniband/hw/mana/qp.c | 10 +++++----- include/uapi/rdma/mana-abi.h | 11 +++++++++++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index e4414d208d4a..9d043f48e272 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -247,6 +247,7 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext, ucontext->doorbell = doorbell_page; ucmd_resp.comp_mask = MANA_IB_UCNTX_ALLOC_PDN_SUPPORT; + ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_EXT_SUPPORT; ret = ib_respond_udata(udata, ucmd_resp); if (ret) return ret; diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index f69866696776..547ddf9ddfea 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -172,7 +172,7 @@ struct mana_ib_cq { enum mana_rc_queue_type { MANA_RC_SEND_QUEUE_REQUESTER = 0, MANA_RC_SEND_QUEUE_RESPONDER, - MANA_RC_SEND_QUEUE_FMR, + MANA_RC_SEND_QUEUE_MMQ, MANA_RC_RECV_QUEUE_REQUESTER, MANA_RC_RECV_QUEUE_RESPONDER, MANA_RC_QUEUE_TYPE_MAX, @@ -372,7 +372,9 @@ struct mana_rnic_destroy_cq_resp { }; /* HW Data */ enum mana_rnic_create_rc_flags { - MANA_RC_FLAG_NO_FMR = 2, + MANA_RC_FLAG_NO_MMQ = BIT(1), + MANA_RC_FLAG_FIXED_SIZE_WQE = BIT(3), + MANA_RC_FLAG_MSN_IN_WQE = BIT(4), }; struct mana_rnic_create_qp_req { diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 8b7be1255c0d..aa5f8c6077bb 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -549,14 +549,14 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext); doorbell = mana_ucontext->doorbell; - flags = MANA_RC_FLAG_NO_FMR; - err = ib_copy_validate_udata_in(udata, ucmd, queue_size); + flags = MANA_RC_FLAG_NO_MMQ; + err = ib_copy_validate_udata_in_cm(udata, ucmd, queue_size, + MANA_IB_RC_QP_FIXED_WQE | MANA_IB_RC_MMQ_CREATE); if (err) return err; for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { - /* skip FMR for user-level RC QPs */ - if (i == MANA_RC_SEND_QUEUE_FMR) { + if (i == MANA_RC_SEND_QUEUE_MMQ) { qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; continue; @@ -580,7 +580,7 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, if (udata) { for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { - if (i == MANA_RC_SEND_QUEUE_FMR) + if (i == MANA_RC_SEND_QUEUE_MMQ) continue; resp.queue_id[j] = qp->rc_qp.queues[i].id; j++; diff --git a/include/uapi/rdma/mana-abi.h b/include/uapi/rdma/mana-abi.h index 32cbbfc80f99..c9a24176660c 100644 --- a/include/uapi/rdma/mana-abi.h +++ b/include/uapi/rdma/mana-abi.h @@ -48,13 +48,23 @@ struct mana_ib_create_qp_resp { __u32 reserved; }; +enum mana_ib_create_rc_qp_flags { + MANA_IB_RC_QP_FIXED_WQE = 1 << 0, + MANA_IB_RC_MMQ_CREATE = 1 << 1, +}; + struct mana_ib_create_rc_qp { __aligned_u64 queue_buf[4]; __u32 queue_size[4]; + __aligned_u64 mmq_buf; + __u32 mmq_size; + __u32 comp_mask; }; struct mana_ib_create_rc_qp_resp { __u32 queue_id[4]; + __u32 mmq_id; + __u32 reserved; }; struct mana_ib_create_uc_qp { @@ -100,6 +110,7 @@ struct mana_ib_create_qp_rss_resp { enum mana_ib_ucontext_support { MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0, + MANA_IB_UCNTX_RC_EXT_SUPPORT = 1 << 1, }; struct mana_ib_alloc_ucontext_resp { From 4309bec53b2496fd527c3d843a5649b91a4f5d8d Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Thu, 27 Aug 2026 07:37:35 -0700 Subject: [PATCH 004/153] RDMA/mana_ib: MMQ creation for RC QPs Create an MMQ for an RC QP when userspace sets the corresponding compatibility-mask flag. Register the userspace MMQ buffer and pass the queue to hardware instead of setting the NO_MMQ creation flag. Keep the MMQ disabled for applications that do not request it, preserving the existing behavior. Return the created MMQ ID to userspace in the extended RC QP response. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/20260827143736.2013093-3-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/qp.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index aa5f8c6077bb..c6821ae5fac1 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -557,16 +557,22 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { if (i == MANA_RC_SEND_QUEUE_MMQ) { - qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; - qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; + if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE) { + flags &= ~MANA_RC_FLAG_NO_MMQ; + err = mana_ib_create_queue(mdev, ucmd.mmq_buf, ucmd.mmq_size, + &qp->rc_qp.queues[i]); + if (err) + goto destroy_queues; + } else { + qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; + qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; + } continue; } err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j], &qp->rc_qp.queues[i]); - if (err) { - ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err); + if (err) goto destroy_queues; - } j++; } @@ -580,8 +586,10 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, if (udata) { for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { - if (i == MANA_RC_SEND_QUEUE_MMQ) + if (i == MANA_RC_SEND_QUEUE_MMQ) { + resp.mmq_id = qp->rc_qp.queues[i].id; continue; + } resp.queue_id[j] = qp->rc_qp.queues[i].id; j++; } From 23d5ff8fd62f2718ac9c6132a77a6282a26d8290 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Thu, 27 Aug 2026 07:37:36 -0700 Subject: [PATCH 005/153] RDMA/mana_ib: fixed send wqe size support for RC QPs Support fixed-size send WQEs for RC QPs when requested by userspace. Calculate the WQE size from the maximum send SGE count and the 32-byte inline OOB size, round it up to a power of two, and pass the size to hardware in basic units. When supported by the adapter, also enable placing the message sequence number in the WQE. Preserve the existing variable-size WQE behavior when userspace does not request fixed-size WQEs. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/20260827143736.2013093-4-kotaranov@linux.microsoft.com Reviewed-by: Long Li Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/main.c | 3 +++ drivers/infiniband/hw/mana/mana_ib.h | 5 ++++- drivers/infiniband/hw/mana/qp.c | 14 ++++++++++++++ include/net/mana/gdma.h | 1 + 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 9d043f48e272..57c6d9faaa81 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -1060,6 +1060,9 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, req.max_recv_sge = attr->cap.max_recv_sge; req.flags = flags; + if (flags & MANA_RC_FLAG_FIXED_SIZE_WQE) + req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu; + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); if (err) return err; diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 547ddf9ddfea..828ca75fd684 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -180,6 +180,7 @@ enum mana_rc_queue_type { struct mana_ib_rc_qp { struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX]; + u32 wqe_size_in_bu; }; enum mana_uc_queue_type { @@ -262,6 +263,7 @@ enum mana_ib_adapter_features { MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4), MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5), MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6), + MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7), }; struct mana_ib_query_adapter_caps_resp { @@ -391,7 +393,8 @@ struct mana_rnic_create_qp_req { u32 max_recv_wr; u32 max_send_sge; u32 max_recv_sge; - u32 reserved; + u8 wqe_size_in_bu; + u8 reserved[3]; }; /* HW Data */ struct mana_rnic_create_qp_resp { diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index c6821ae5fac1..29bfee9ac997 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -422,6 +422,11 @@ static u32 mana_ib_wqe_size(u32 sge, u32 oob_size) return ALIGN(wqe_size, GDMA_WQE_BU_SIZE); } +static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size) +{ + return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size)); +} + static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type) { u32 queue_size; @@ -576,6 +581,15 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, j++; } + if (ucmd.comp_mask & MANA_IB_RC_QP_FIXED_WQE) { + u32 wqe_size = mana_ib_fixed_wqe_size(attr->cap.max_send_sge, + INLINE_OOB_EXTRA_LARGE_SIZE); + flags |= MANA_RC_FLAG_FIXED_SIZE_WQE; + if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT) + flags |= MANA_RC_FLAG_MSN_IN_WQE; + qp->rc_qp.wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; + } + err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create rc qp %d\n", err); diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 308950f9b54b..06af65d50eb7 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -552,6 +552,7 @@ struct gdma_wqe { #define INLINE_OOB_SMALL_SIZE 8 #define INLINE_OOB_LARGE_SIZE 24 +#define INLINE_OOB_EXTRA_LARGE_SIZE 32 #define MANA_MAX_TX_WQE_SGL_ENTRIES 30 From 19881618ffa454c63e6b6bf34acd7cac28c695dc Mon Sep 17 00:00:00 2001 From: xiongweimin Date: Thu, 16 Jul 2026 13:52:55 +0800 Subject: [PATCH 006/153] RDMA/rxe: Hold netdev reference for transmit skbs rxe_init_packet() assigns skb->dev from an RCU-protected GID attribute without holding a netdev reference. If the netdev is unregistered before the skb is freed, subsequent accesses to skb->dev are unsafe. Hold a reference with dev_hold() when the skb is initialized and release it from the transmit destructor or via rxe_put_skb() on error paths that run before the destructor is installed. skb->dev can change on the TX path (VLAN/bond/tunnel, ip_finish_output2, etc.), so put must use the same netdev that was held. Stash that pointer in skb_shinfo()->destructor_arg: skb->cb is already used by rxe_pkt_info and is rewritten by IP control blocks. To avoid blocking netdev unregistration on held skbs, flush all QPs to the error state on NETDEV_GOING_DOWN and NETDEV_UNREGISTER so pending TX work is drained and references can be dropped. Signed-off-by: xiongweimin Link: https://patch.msgid.link/20260716055255.157674-1-xiongwm2026@163.com Reviewed-by: Zhu Yanjun Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_loc.h | 1 + drivers/infiniband/sw/rxe/rxe_net.c | 68 ++++++++++++++++++++++++++-- drivers/infiniband/sw/rxe/rxe_req.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 4 +- 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 64d636bf80fd..7c3cc48e845c 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -92,6 +92,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem); /* rxe_net.c */ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, int paylen, struct rxe_pkt_info *pkt); +void rxe_put_skb(struct sk_buff *skb); int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt, struct sk_buff *skb); int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 53daaf4c1eb2..c66ae71d970d 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -429,6 +429,29 @@ int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt, return err; } +/* + * skb->dev may be rewritten on the TX path (VLAN/bond/tunnel, etc.). + * The netdev we held in rxe_init_packet() is kept in destructor_arg so + * that put always matches hold. skb->cb cannot be used: it is already + * occupied by struct rxe_pkt_info and rewritten by IP control blocks. + */ +static void rxe_skb_set_held_ndev(struct sk_buff *skb, struct net_device *ndev) +{ + dev_hold(ndev); + skb->dev = ndev; + skb_shinfo(skb)->destructor_arg = ndev; +} + +static void rxe_skb_put_held_ndev(struct sk_buff *skb) +{ + struct net_device *ndev = skb_shinfo(skb)->destructor_arg; + + if (ndev) { + skb_shinfo(skb)->destructor_arg = NULL; + dev_put(ndev); + } +} + static void rxe_skb_tx_dtor(struct sk_buff *skb) { struct rxe_qp *qp = skb->sk->sk_user_data; @@ -441,6 +464,18 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb) rxe_put(qp); sock_put(skb->sk); + rxe_skb_put_held_ndev(skb); +} + +/* + * Free an skb that still holds a netdev reference from rxe_init_packet() + * and does not yet have rxe_skb_tx_dtor() installed. Once the TX + * destructor is set, callers must use kfree_skb() instead. + */ +void rxe_put_skb(struct sk_buff *skb) +{ + rxe_skb_put_held_ndev(skb); + kfree_skb(skb); } static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt) @@ -529,7 +564,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, goto done; drop: - kfree_skb(skb); + rxe_put_skb(skb); err = 0; done: return err; @@ -574,8 +609,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, skb_reserve(skb, hdr_len + LL_RESERVED_SPACE(ndev)); - /* FIXME: hold reference to this netdev until life of this skb. */ - skb->dev = ndev; + rxe_skb_set_held_ndev(skb, ndev); rcu_read_unlock(); if (av->network_type == RXE_NETWORK_TYPE_IPV4) @@ -710,6 +744,28 @@ void rxe_set_port_state(struct rxe_dev *rxe) dev_put(ndev); } +/* + * Move all QPs to the error state so pending send/recv work is drained and + * in-flight TX skbs (which hold a netdev reference) can be released. Called + * from the netdev notifier so unregister cannot stall on held skbs. + */ +static void rxe_flush_qps(struct rxe_dev *rxe) +{ + struct rxe_pool_elem *elem; + struct rxe_qp *qp; + unsigned long index; + + rcu_read_lock(); + xa_for_each(&rxe->qp_pool.xa, index, elem) { + if (!elem || !kref_get_unless_zero(&elem->ref_cnt)) + continue; + qp = elem->obj; + rxe_qp_error(qp); + rxe_put(qp); + } + rcu_read_unlock(); +} + static int rxe_notify(struct notifier_block *not_blk, unsigned long event, void *arg) @@ -721,7 +777,12 @@ static int rxe_notify(struct notifier_block *not_blk, return NOTIFY_OK; switch (event) { + case NETDEV_GOING_DOWN: + /* Start draining TX queues before the netdev disappears. */ + rxe_flush_qps(rxe); + break; case NETDEV_UNREGISTER: + rxe_flush_qps(rxe); ib_unregister_device_queued(&rxe->ib_dev); rxe_net_del(&rxe->ib_dev); break; @@ -735,7 +796,6 @@ static int rxe_notify(struct notifier_block *not_blk, rxe_counter_inc(rxe, RXE_CNT_LINK_DOWNED); break; case NETDEV_REBOOT: - case NETDEV_GOING_DOWN: case NETDEV_CHANGEADDR: case NETDEV_CHANGENAME: case NETDEV_FEAT_CHANGE: diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 24f5c044363f..53f7c1e7fe55 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -811,7 +811,7 @@ int rxe_requester(struct rxe_qp *qp) wqe->status = IB_WC_LOC_PROT_ERR; else wqe->status = IB_WC_LOC_QP_OP_ERR; - kfree_skb(skb); + rxe_put_skb(skb); if (ah) rxe_put(ah); goto err; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 02b16e2b49b8..b1ea720aa34f 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -866,7 +866,7 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, err = rxe_prepare(&qp->pri_av, ack, skb); if (err) { - kfree_skb(skb); + rxe_put_skb(skb); return NULL; } @@ -994,7 +994,7 @@ static enum resp_states read_reply(struct rxe_qp *qp, err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt), payload, RXE_FROM_MR_OBJ); if (err) { - kfree_skb(skb); + rxe_put_skb(skb); state = RESPST_ERR_RKEY_VIOLATION; goto err_out; } From 901c15a2099d43538173fd82ba5efb55ab842e75 Mon Sep 17 00:00:00 2001 From: Ruizhe Zhou Date: Wed, 12 Aug 2026 11:36:46 +0800 Subject: [PATCH 007/153] RDMA/hfi1: Remove obsolete 32-bit DMA mask fallback The DMA API guarantees support for masks of 32 bits or wider and explicitly identifies retrying a 32-bit mask after a 64-bit request as incorrect: https://docs.kernel.org/core-api/dma-api-howto.html#dma-addressing-capabilities Remove the obsolete fallback while retaining the error check so that a genuine DMA setup failure is still reported and aborts initialization. Signed-off-by: Ruizhe Zhou Link: https://patch.msgid.link/20260812033647.2281547-2-zhouruizhe@resnics.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hfi1/pcie.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index 1154b8cc713c..106bed4932c8 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -53,17 +53,9 @@ int hfi1_pcie_init(struct pci_dev *pdev) ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (ret) { - /* - * If the 64 bit setup fails, try 32 bit. Some systems - * do not setup 64 bit maps on systems with 2GB or less - * memory installed. - */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (ret) { - dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n", - ERR_PTR(ret)); - goto bail; - } + dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n", + ERR_PTR(ret)); + goto bail; } pci_set_master(pdev); From 08a978db86e877172b8a2b9ae6fd7823ecbf72bb Mon Sep 17 00:00:00 2001 From: Ruizhe Zhou Date: Wed, 12 Aug 2026 11:36:47 +0800 Subject: [PATCH 008/153] RDMA/mlx5: Remove obsolete 32-bit DMA mask fallback The DMA API guarantees support for masks of 32 bits or wider and explicitly identifies retrying a 32-bit mask after a 64-bit request as incorrect: https://docs.kernel.org/core-api/dma-api-howto.html#dma-addressing-capabilities Remove the obsolete fallback from data direct while retaining the error check so that a genuine DMA setup failure is still reported and aborts initialization. Signed-off-by: Ruizhe Zhou Link: https://patch.msgid.link/20260812033647.2281547-3-zhouruizhe@resnics.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index d57484245c38..93a56ee2c4ef 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -65,13 +65,9 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (err) { - dev_warn(&pdev->dev, - "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err); - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err); - return err; - } + dev_err(&pdev->dev, + "Can't set 64-bit PCI DMA mask, err=%d\n", err); + return err; } dma_set_max_seg_size(&pdev->dev, SZ_2G); From 0e7d08160f6127052ef8f2093f519a748772cf6e Mon Sep 17 00:00:00 2001 From: Snehal Sanghvi Date: Wed, 12 Aug 2026 03:05:31 -0700 Subject: [PATCH 009/153] RDMA/mana_ib: Enable multi-port GSI QP support for mana_ib Add MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT so mana_ib can create a GSI QP per IB port instead of just one. When the feature is negotiated, the port's vNIC MAC is passed on GSI QP creation and each GSI QP is indexed in the QP table by port so the GSI SQ drain can reach every port. Also move MANA_SENDQ_MASK to BIT(0), freeing the top byte of the queue-id key to index GSI QPs by port and scaling the feature to the full 8-bit port range. Signed-off-by: Snehal Sanghvi Signed-off-by: Konstantin Taranov Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 26 +++++++++++++++++--------- drivers/infiniband/hw/mana/main.c | 28 ++++++++++++++++++++++++++-- drivers/infiniband/hw/mana/mana_ib.h | 15 +++++++++++++-- drivers/infiniband/hw/mana/qp.c | 17 ++++++++++++++++- 4 files changed, 72 insertions(+), 14 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index d4e5e3f91268..6f9ac8b4aac8 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -293,18 +293,12 @@ static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc return wc_index; } -void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) +static void mana_drain_gsi_sq(struct mana_ib_qp *qp) { - struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false); + struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); struct ud_sq_shadow_wqe *shadow_wqe; - struct mana_ib_cq *cq; unsigned long flags; - if (!qp) - return; - - cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); - spin_lock_irqsave(&cq->cq_lock, flags); while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) != NULL) { @@ -315,8 +309,22 @@ void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) if (cq->ibcq.comp_handler) cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); +} - mana_put_qp_ref(qp); +void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) +{ + struct mana_ib_qp *qp; + u32 port; + + /* One GSI QP per port, indexed in the QP table by (port << 24 | MANA_GSI_QPN) */ + for (port = 1; port <= mdev->ib_dev.phys_port_cnt; port++) { + qp = mana_get_qp_ref(mdev, (port << 24) | MANA_GSI_QPN, false); + if (!qp) + continue; + + mana_drain_gsi_sq(qp); + mana_put_qp_ref(qp); + } } int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 57c6d9faaa81..83a97f1c5caa 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -593,7 +593,11 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, immutable->gid_tbl_len = attr.gid_tbl_len; if (mana_ib_is_rnic(dev)) { - if (port_num == 1) { + bool port_supports_cm = (port_num == 1 || + (dev->adapter_caps.feature_flags & + MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)); + + if (port_supports_cm) { immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; immutable->max_mad_size = IB_MGMT_MAD_SIZE; } else { @@ -671,10 +675,14 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width); props->pkey_tbl_len = 1; if (mana_ib_is_rnic(dev)) { + bool port_supports_cm = (port == 1 || + (dev->adapter_caps.feature_flags & + MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)); + props->gid_tbl_len = 16; props->ip_gids = true; props->max_msg_sz = SZ_16M; - if (port == 1) + if (port_supports_cm) props->port_cap_flags = IB_PORT_CM_SUP; } @@ -1138,6 +1146,7 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, struct gdma_context *gc = mdev_to_gc(mdev); struct mana_rnic_create_udqp_resp resp = {}; struct mana_rnic_create_udqp_req req = {}; + struct net_device *ndev; int err, i; mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(resp)); @@ -1154,6 +1163,21 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, req.max_send_sge = attr->cap.max_send_sge; req.max_recv_sge = attr->cap.max_recv_sge; req.qp_type = type; + + /* For GSI QPs, pass the vNIC MAC so the SoC can associate the QP with + * the correct port, transition to INIT, and allocate a per-port QPN. + * MAC must be in reversed byte order. + */ + if (type == IB_QPT_GSI && + (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)) { + ndev = mana_ib_get_netdev(&mdev->ib_dev, attr->port_num); + if (ndev) { + copy_in_reverse(req.mac, ndev->dev_addr, ETH_ALEN); + req.flags = MANA_UD_QP_FLAG_CREATE_IN_INIT; + req.hdr.req.msg_version = GDMA_MESSAGE_V2; + } + } + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); if (err) return err; diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 828ca75fd684..1be33ed8bd3b 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -24,8 +24,12 @@ /* MANA doesn't have any limit for MR size */ #define MANA_IB_MAX_MR_SIZE U64_MAX -/* Send queue ID mask */ -#define MANA_SENDQ_MASK BIT(31) +/* + * Send queue ID mask. Queue IDs are 2-bit aligned (see MANA_QID_SUBTYPE_MASK), + * so bit 0 is always free to tag send queues in the lookup table. This keeps + * the whole top byte available to index per-port GSI QPs by (port << 24). + */ +#define MANA_SENDQ_MASK BIT(0) /* Queue ID encodes type in the lower 2 bits */ #define MANA_QID_SUBTYPE_MASK 0x3 @@ -264,6 +268,7 @@ enum mana_ib_adapter_features { MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5), MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6), MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7), + MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT = BIT(15), }; struct mana_ib_query_adapter_caps_resp { @@ -450,8 +455,14 @@ struct mana_rnic_create_udqp_req { u32 max_recv_wr; u32 max_send_sge; u32 max_recv_sge; + u8 mac[ETH_ALEN]; /* V2: port MAC for multi-port GSI */ + u16 flags; /* V2: MANA_UD_QP_FLAG_* */ }; /* HW Data */ +enum mana_ud_qp_flags { + MANA_UD_QP_FLAG_CREATE_IN_INIT = BIT(0), +}; + struct mana_rnic_create_udqp_resp { struct gdma_resp_hdr hdr; mana_handle_t qp_handle; diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 29bfee9ac997..fac43b3a5eb7 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -514,8 +514,20 @@ static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) if (err) goto err_remove_sq; + /* GSI QPs are additionally indexed by (port << 24 | MANA_GSI_QPN) so the + * per-port GSI SQ drain can find each of them by iterating ports. + */ + if (qp->ibqp.qp_type == IB_QPT_GSI) { + err = mana_table_store_qp_qid(mdev, qp, + (qp->port << 24) | MANA_GSI_QPN, false); + if (err) + goto err_remove_rq; + } + return 0; +err_remove_rq: + mana_table_remove_qp_qid(mdev, rq->id, false); err_remove_sq: mana_table_remove_qp_qid(mdev, sq->id, true); mana_table_drain_qp_ref(qp); @@ -534,6 +546,8 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp mana_table_remove_qp_qid(mdev, sq->id, true); mana_table_remove_qp_qid(mdev, rq->id, false); + if (qp->ibqp.qp_type == IB_QPT_GSI) + mana_table_remove_qp_qid(mdev, (qp->port << 24) | MANA_GSI_QPN, false); mana_table_drain_qp_ref(qp); } @@ -751,7 +765,8 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_err(&mdev->ib_dev, "Failed to create ud qp %d\n", err); goto destroy_shadow_queues; } - qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; + qp->ibqp.qp_num = (qp->ibqp.qp_type == IB_QPT_GSI) ? + MANA_GSI_QPN : qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; qp->port = attr->port_num; for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) From 15c4aa405a3dc4b874caea98b4f5db07236597ab Mon Sep 17 00:00:00 2001 From: Liu Zhenlong Date: Wed, 19 Aug 2026 00:35:13 +0800 Subject: [PATCH 010/153] RDMA/rtrs-clt: use find_next_zero_bit() for permit allocation __rtrs_get_permit() scans permits_map with find_first_zero_bit() and claims the bit with test_and_set_bit_lock(), restarting from bit 0 on a lost race. Use find_next_zero_bit() to resume from the last position so a lost race does not rescan the already-set low bits; on reaching the end, wrap to the beginning to exhaust the map. Compile-tested: arm64 defconfig + INFINIBAND_RTRS_CLIENT=m, rtrs-clt.o Reviewed-by: Jack Wang Signed-off-by: Liu Zhenlong Link: https://patch.msgid.link/20260818163513.53875-1-dragonliu2018@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 7b2c51ae614f..f48872bb0024 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -70,19 +70,24 @@ __rtrs_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type) { size_t max_depth = clt->queue_depth; struct rtrs_permit *permit; - int bit; + unsigned long bit = 0; /* - * Adapted from null_blk get_tag(). Callers from different cpus may - * grab the same bit, since find_first_zero_bit is not atomic. - * But then the test_and_set_bit_lock will fail for all the - * callers but one, so that they will loop again. - * This way an explicit spinlock is not required. + * Callers from different CPUs may grab the same bit, since the bitmap + * scan is not atomic. But then the test_and_set_bit_lock() will fail + * for all the callers but one, so that they loop again. This way an + * explicit spinlock is not required. find_next_zero_bit() resumes + * from the last position so that a lost race does not rescan the + * already-set low bits; if it reaches the end, wrap to the beginning + * to exhaust the map and still find a permit freed below the cursor. */ do { - bit = find_first_zero_bit(clt->permits_map, max_depth); - if (bit >= max_depth) - return NULL; + bit = find_next_zero_bit(clt->permits_map, max_depth, bit); + if (bit >= max_depth) { + bit = find_first_zero_bit(clt->permits_map, max_depth); + if (bit >= max_depth) + return NULL; + } } while (test_and_set_bit_lock(bit, clt->permits_map)); permit = get_permit(clt, bit); From 3547f46c31993d41748f0f7936f9d4dc5b5d4b12 Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Mon, 31 Aug 2026 10:03:23 +0800 Subject: [PATCH 011/153] RDMA/hns: Fix missing CQE when UD QP use different SL Due to the hardware contraint, CQEs may be dropped when a UD QP sends with multiple SLs. Pin the SL to the value from the first post_send on the QP to avoid this issue. Fixes: 66d86e529dd5 ("RDMA/hns: Add UD support for HIP09") Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260831020324.3540901-2-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_device.h | 1 + drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index eb7b1865e4c7..95529251e2e2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -654,6 +654,7 @@ struct hns_roce_qp { u8 priority; spinlock_t flush_lock; struct hns_roce_dip *dip; + bool ud_sl_set; }; struct hns_roce_ib_iboe { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 27cc7df55ee7..fd198faf4ee4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -431,7 +431,8 @@ static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, return 0; } -static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, +static int fill_ud_av(struct hns_roce_qp *qp, + struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, struct hns_roce_ah *ah) { struct ib_device *ib_dev = ah->ibah.device; @@ -441,7 +442,12 @@ static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel); - hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl); + if (!qp->ud_sl_set) { + qp->sl = ah->av.sl; + qp->ud_sl_set = true; + } + + hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, qp->sl); ud_sq_wqe->sgid_index = ah->av.gid_index; @@ -491,12 +497,10 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp, qp->qkey : ud_wr(wr)->remote_qkey); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn); - ret = fill_ud_av(ud_sq_wqe, ah); + ret = fill_ud_av(qp, ud_sq_wqe, ah); if (ret) return ret; - qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl; - set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge); /* @@ -5603,6 +5607,7 @@ static void v2_set_flushed_fields(struct ib_qp *ibqp, hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head); hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX); hr_qp->state = IB_QPS_ERR; + hr_qp->ud_sl_set = false; spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag); if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */ From 905dc5f0d98e3c081aa36d81989ec7a69df70d92 Mon Sep 17 00:00:00 2001 From: Chengchang Tang Date: Mon, 31 Aug 2026 10:03:24 +0800 Subject: [PATCH 012/153] RDMA/hns: Support setting GSI QP SL via debugfs Due to the hardware contraint, CQEs may be dropped when a UD QP sends with multiple SLs. For GSI QP, provide debugfs to allow users to set a fixed SL. This is only allowed when the device is link down. Example: # cat /sys/kernel/debug/hns_roce//gsi_sl 0 # echo 3 > /sys/kernel/debug/hns_roce//gsi_sl # cat /sys/kernel/debug/hns_roce//gsi_sl 3 Signed-off-by: Chengchang Tang Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260831020324.3540901-3-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_debugfs.c | 55 ++++++++++++++++++++ drivers/infiniband/hw/hns/hns_roce_debugfs.h | 1 + drivers/infiniband/hw/hns/hns_roce_device.h | 1 + drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 5 +- 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index 05630f7c9155..2487a80b689a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -339,6 +339,60 @@ static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev, } } +static int gsi_sl_debugfs_show(struct seq_file *file, void *offset) +{ + struct hns_roce_dev *hr_dev = file->private; + + seq_printf(file, "%u\n", hr_dev->gsi_sl); + + return 0; +} + +static ssize_t gsi_sl_debugfs_store(char *buf, size_t count, void *data) +{ + struct hns_roce_dev *hr_dev = data; + struct net_device *netdev; + int ret; + u8 val; + + netdev = ib_device_get_netdev(&hr_dev->ib_dev, 1); + if (!netdev) + return -ENODEV; + + if (ib_get_curr_port_state(netdev) == IB_PORT_ACTIVE) { + ret = -EOPNOTSUPP; + goto out; + } + + ret = kstrtou8(buf, 0, &val); + if (ret) + goto out; + + if (!check_sl_valid(hr_dev, val)) { + ret = -EINVAL; + goto out; + } + + hr_dev->gsi_sl = val; + +out: + dev_put(netdev); + return ret ? : count; +} + +static void create_gsi_sl_debugfs(struct hns_roce_dev *hr_dev, + struct dentry *parent) +{ + struct hns_debugfs_seqfile *seqfile = &hr_dev->dbgfs.gsi_sl; + + seqfile->read = gsi_sl_debugfs_show; + seqfile->write = gsi_sl_debugfs_store; + seqfile->data = hr_dev; + + debugfs_create_file("gsi_sl", 0600, parent, seqfile, + &hns_debugfs_seqfile_fops); +} + /* debugfs for device */ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev) { @@ -349,6 +403,7 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev) create_sw_stat_debugfs(hr_dev, dbgfs->root); create_cc_param_debugfs(hr_dev, dbgfs->root); + create_gsi_sl_debugfs(hr_dev, dbgfs->root); } void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev) diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h index 116b1e8b6677..286704711785 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.h +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.h @@ -47,6 +47,7 @@ struct hns_roce_dev_debugfs { struct dentry *root; struct hns_sw_stat_debugfs sw_stat_root; struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM]; + struct hns_debugfs_seqfile gsi_sl; }; struct hns_roce_dev; diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 95529251e2e2..f4f899e87ea6 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1048,6 +1048,7 @@ struct hns_roce_dev { struct hns_roce_dev_debugfs dbgfs; atomic64_t *dfx_cnt; struct hns_roce_scc_param *scc_param; + u8 gsi_sl; }; enum hns_roce_trace_type { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index fd198faf4ee4..368e1d74c283 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -442,8 +442,9 @@ static int fill_ud_av(struct hns_roce_qp *qp, hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel); - if (!qp->ud_sl_set) { - qp->sl = ah->av.sl; + if (!qp->ud_sl_set || qp->ibqp.qp_type == IB_QPT_GSI) { + qp->sl = qp->ibqp.qp_type == IB_QPT_GSI ? + hr_dev->gsi_sl : ah->av.sl; qp->ud_sl_set = true; } From 89b48ad5671fa082645f1be7e933236ac21de974 Mon Sep 17 00:00:00 2001 From: Snehal Sanghvi Date: Wed, 2 Sep 2026 08:38:13 -0700 Subject: [PATCH 013/153] RDMA/mana_ib: advertise pow2 RC SQ support to user space The send queue of an RC QP normally needs an extra rollback area in shared memory. With fixed-size WQEs the rollback can be derived from the WQE size alone, so an adapter supporting it can accept a plain power-of-two send queue and the rollback area becomes unnecessary. Report the adapter capability to user space with a new comp_mask bit in the alloc ucontext response, so that user space can request a power-of-two RC send queue when the adapter supports it. Signed-off-by: Snehal Sanghvi Link: https://patch.msgid.link/20260902153813.3385076-1-snsanghvi@linux.microsoft.com Reviewed-by: Konstantin Taranov Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/main.c | 2 ++ drivers/infiniband/hw/mana/mana_ib.h | 1 + include/uapi/rdma/mana-abi.h | 1 + 3 files changed, 4 insertions(+) diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 83a97f1c5caa..672bd3f650dd 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -248,6 +248,8 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext, ucontext->doorbell = doorbell_page; ucmd_resp.comp_mask = MANA_IB_UCNTX_ALLOC_PDN_SUPPORT; ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_EXT_SUPPORT; + if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT) + ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_SQ_POW2_SUPPORT; ret = ib_respond_udata(udata, ucmd_resp); if (ret) return ret; diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 1be33ed8bd3b..4da110b302c7 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -268,6 +268,7 @@ enum mana_ib_adapter_features { MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5), MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6), MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7), + MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT = BIT(14), MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT = BIT(15), }; diff --git a/include/uapi/rdma/mana-abi.h b/include/uapi/rdma/mana-abi.h index c9a24176660c..ebc9a655b826 100644 --- a/include/uapi/rdma/mana-abi.h +++ b/include/uapi/rdma/mana-abi.h @@ -111,6 +111,7 @@ struct mana_ib_create_qp_rss_resp { enum mana_ib_ucontext_support { MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0, MANA_IB_UCNTX_RC_EXT_SUPPORT = 1 << 1, + MANA_IB_UCNTX_RC_SQ_POW2_SUPPORT = 1 << 2, }; struct mana_ib_alloc_ucontext_resp { From 0ff7fcd6aac16dd149ad322687a10c8ea7d806b2 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Thu, 3 Sep 2026 16:25:50 +0800 Subject: [PATCH 014/153] RDMA/nldev: Put the device when dellink fails nldev_dellink() takes a device reference through ib_device_get_by_index() and normally hands it over to ib_unregister_device_and_put(). The error path of the ->dellink callback returns without releasing it, so the reference is leaked. Once that happens, any later unregistration of the device blocks forever, because disable_device() drops its own reference and then waits for the refcount to drain. No in-tree driver returns an error from ->dellink today, so the leak is currently latent. Put the device before returning the error. Fixes: a60e3f3d6fba ("RDMA/nldev: Add dellink function pointer") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260903082550.2257-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/nldev.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index a4014a230639..a1542ed80fcc 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -1895,8 +1895,10 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh, mutex_lock(&nldev_dellink_mutex); err = device->link_ops->dellink(device); mutex_unlock(&nldev_dellink_mutex); - if (err) + if (err) { + ib_device_put(device); return err; + } } ib_unregister_device_and_put(device); From c470050f2a7a7748ce35367e828885a8e658d712 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 6 Sep 2026 03:52:38 -0400 Subject: [PATCH 015/153] RDMA/irdma: Preserve fast-registration IOVA on 32-bit ib_mr::iova is u64, but the fast-registration path passes it through void * and uintptr_t. These conversions truncate the upper 32 bits on 32-bit kernels before the WQE is built. Store the IOVA as u64 and write it directly to the WQE. The sole caller always uses VA-based addressing, so remove the unused FBO selection and set the VA-based bit unconditionally. Fixes: b48c24c2d710 ("RDMA/irdma: Implement device supported verb APIs") Link: https://patch.msgid.link/20260903-64-bit-iova-is-silently-truncated-to-v1-1-96e878ea6873@nvidia.com Reviewed-by: Jacob Moroni Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/ctrl.c | 6 ++---- drivers/infiniband/hw/irdma/type.h | 4 +--- drivers/infiniband/hw/irdma/verbs.c | 3 +-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c index 335ae3c82e17..7af1c68b2fb2 100644 --- a/drivers/infiniband/hw/irdma/ctrl.c +++ b/drivers/infiniband/hw/irdma/ctrl.c @@ -1723,9 +1723,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, info->wr_id, wqe_idx, &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); - temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? - (uintptr_t)info->va : info->fbo; - set_64bit_val(wqe, 0, temp); + set_64bit_val(wqe, 0, info->va); temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI, info->first_pm_pbl_index >> 16); @@ -1742,7 +1740,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) | FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) | FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) | - FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) | + IRDMAQPSQ_VABASEDTO | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h index 5557d9338796..81c5c5da1988 100644 --- a/drivers/infiniband/hw/irdma/type.h +++ b/drivers/infiniband/hw/irdma/type.h @@ -1173,13 +1173,11 @@ struct irdma_reg_ns_stag_info { struct irdma_fast_reg_stag_info { u64 wr_id; u64 reg_addr_pa; - u64 fbo; - void *va; + u64 va; u64 total_len; u32 page_size; u32 chunk_size; u32 first_pm_pbl_index; - enum irdma_addressing_type addr_type; irdma_stag_index stag_idx; u16 access_rights; u32 pd_id; diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 9cfd84dd6869..c0a9fc862d43 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -4276,8 +4276,7 @@ static int irdma_post_send(struct ib_qp *ibqp, stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; stag_info.page_size = reg_wr(ib_wr)->mr->page_size; stag_info.wr_id = ib_wr->wr_id; - stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; - stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; + stag_info.va = iwmr->ibmr.iova; stag_info.total_len = iwmr->ibmr.length; stag_info.reg_addr_pa = *palloc->level1.addr; stag_info.first_pm_pbl_index = palloc->level1.idx; From 4a93466061d38f159b857aad63a9cc3cd3affadb Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 6 Sep 2026 03:52:38 -0400 Subject: [PATCH 016/153] RDMA/irdma: Remove unused post_sq arguments The only callers of irdma_sc_qp_flush_wqes() and irdma_sc_mr_fast_register() always request their respective send queues to be posted, so their post_sq false branches are unreachable. Remove both arguments and post the queues unconditionally. Drop the redundant QP flush request assignments as well. Link: https://patch.msgid.link/20260903-64-bit-iova-is-silently-truncated-to-v1-2-96e878ea6873@nvidia.com Reviewed-by: Kalesh AP Reviewed-by: Jacob Moroni Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/ctrl.c | 17 +++++------------ drivers/infiniband/hw/irdma/hw.c | 2 -- drivers/infiniband/hw/irdma/protos.h | 3 +-- drivers/infiniband/hw/irdma/type.h | 3 +-- drivers/infiniband/hw/irdma/verbs.c | 4 ++-- 5 files changed, 9 insertions(+), 20 deletions(-) diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c index 7af1c68b2fb2..0db73fd7d01b 100644 --- a/drivers/infiniband/hw/irdma/ctrl.c +++ b/drivers/infiniband/hw/irdma/ctrl.c @@ -1689,11 +1689,9 @@ static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev, * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp * @qp: sc qp struct * @info: fast mr info - * @post_sq: flag for cqp db to ring */ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, - struct irdma_fast_reg_stag_info *info, - bool post_sq) + struct irdma_fast_reg_stag_info *info) { u64 temp, hdr; __le64 *wqe; @@ -1753,8 +1751,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe, IRDMA_QP_WQE_MIN_SIZE, false); - if (post_sq) - irdma_uk_qp_post_wr(&qp->qp_uk); + irdma_uk_qp_post_wr(&qp->qp_uk); return 0; } @@ -2623,11 +2620,9 @@ static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp, * @qp: sc qp * @info: dlush information * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring */ int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, - struct irdma_qp_flush_info *info, u64 scratch, - bool post_sq) + struct irdma_qp_flush_info *info, u64 scratch) { u64 temp = 0; __le64 *wqe; @@ -2695,8 +2690,7 @@ int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); - if (post_sq) - irdma_sc_cqp_post_sq(cqp); + irdma_sc_cqp_post_sq(cqp); return 0; } @@ -6112,8 +6106,7 @@ static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev, case IRDMA_OP_QP_FLUSH_WQES: status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp, &pcmdinfo->in.u.qp_flush_wqes.info, - pcmdinfo->in.u.qp_flush_wqes.scratch, - pcmdinfo->post_sq); + pcmdinfo->in.u.qp_flush_wqes.scratch); break; case IRDMA_OP_GEN_AE: status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp, diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index c345cc654256..cae4f316def4 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -2742,7 +2742,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, hw_info = &cqp_request->info.in.u.qp_flush_wqes.info; memcpy(hw_info, info, sizeof(*hw_info)); cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES; - cqp_info->post_sq = 1; cqp_info->in.u.qp_flush_wqes.qp = qp; cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(rf, cqp_request); @@ -2789,7 +2788,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, hw_info = &new_req->info.in.u.qp_flush_wqes.info; memcpy(hw_info, info, sizeof(*hw_info)); cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES; - cqp_info->post_sq = 1; cqp_info->in.u.qp_flush_wqes.qp = qp; cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)new_req; diff --git a/drivers/infiniband/hw/irdma/protos.h b/drivers/infiniband/hw/irdma/protos.h index 324cfbf21764..561a0287b945 100644 --- a/drivers/infiniband/hw/irdma/protos.h +++ b/drivers/infiniband/hw/irdma/protos.h @@ -19,8 +19,7 @@ void irdma_sc_rt_init(struct irdma_sc_dev *dev); void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp); __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch); int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, - struct irdma_fast_reg_stag_info *info, - bool post_sq); + struct irdma_fast_reg_stag_info *info); /* HMC/FPM functions */ int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id); /* stats misc */ diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h index 81c5c5da1988..8b4d2c0b283f 100644 --- a/drivers/infiniband/hw/irdma/type.h +++ b/drivers/infiniband/hw/irdma/type.h @@ -1381,8 +1381,7 @@ int irdma_sc_qp_create(struct irdma_sc_qp *qp, int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq); int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, - struct irdma_qp_flush_info *info, u64 scratch, - bool post_sq); + struct irdma_qp_flush_info *info, u64 scratch); int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info); int irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info, u64 scratch, diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index c0a9fc862d43..7d3540824e3a 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -4283,8 +4283,8 @@ static int irdma_post_send(struct ib_qp *ibqp, stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) stag_info.chunk_size = 1; - err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, - true); + err = irdma_sc_mr_fast_register(&iwqp->sc_qp, + &stag_info); break; } default: From 63367df6e7255067ad6a83abe0d2799dfa491876 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 5 Sep 2026 11:20:57 +0200 Subject: [PATCH 017/153] ABI: sysfs-class-infiniband: Drop the entries of the removed qib driver Commit 24baad32b710 ("RDMA/qib: Remove outdated driver") removed the qib driver but left its /sys/class/infiniband/qibX/* entries in the stable ABI file. Nothing creates those files any more. Remove the block. Fixes: 24baad32b710 ("RDMA/qib: Remove outdated driver") Assisted-by: LLM Signed-off-by: Karl Mehltretter Link: https://patch.msgid.link/20260905092057.41306-1-kmehltretter@gmail.com Signed-off-by: Leon Romanovsky --- .../ABI/stable/sysfs-class-infiniband | 118 ------------------ 1 file changed, 118 deletions(-) diff --git a/Documentation/ABI/stable/sysfs-class-infiniband b/Documentation/ABI/stable/sysfs-class-infiniband index 7ba116103429..726a38470e58 100644 --- a/Documentation/ABI/stable/sysfs-class-infiniband +++ b/Documentation/ABI/stable/sysfs-class-infiniband @@ -422,124 +422,6 @@ Description: information) =============== ============================================= - -sysfs interface for Intel IB driver qib ---------------------------------------- - -What: /sys/class/infiniband/qibX/version -What: /sys/class/infiniband/qibX/hw_rev -What: /sys/class/infiniband/qibX/hca_type -What: /sys/class/infiniband/qibX/board_id -What: /sys/class/infiniband/qibX/boardversion -What: /sys/class/infiniband/qibX/nctxts -What: /sys/class/infiniband/qibX/localbus_info -What: /sys/class/infiniband/qibX/tempsense -What: /sys/class/infiniband/qibX/serial -What: /sys/class/infiniband/qibX/nfreectxts -What: /sys/class/infiniband/qibX/chip_reset -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - =============== ====================================================== - version: (RO) Display version information of installed software - and drivers. - - hw_rev: (RO) Hardware revision number - - hca_type: (RO) Host channel adapter type - - board_id: (RO) Manufacturing board id - - boardversion: (RO) Current version of the chip architecture - - nctxts: (RO) Return the number of user ports (contexts) - available - - localbus_info: (RO) Human readable localbus info - - tempsense: (RO) Display temp sense registers in decimal - - serial: (RO) Serial number of the HCA - - nfreectxts: (RO) The number of free user ports (contexts) - available. - - chip_reset: (WO) Reset the chip if possible by writing - "reset" to this file. Only allowed if no user - contexts are open that use chip resources. - =============== ====================================================== - - -What: /sys/class/infiniband/qibX/ports//sl2vl/[0-15] -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - (RO) The directory contains 16 files numbered 0-15 that specify - the Service Level (SL). Listing the SL files returns the Virtual - Lane (VL) as programmed by the SL. - -What: /sys/class/infiniband/qibX/ports//CCMgtA/cc_settings_bin -What: /sys/class/infiniband/qibX/ports//CCMgtA/cc_table_bin -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - Per-port congestion control. Both are binary attributes. - - =============== ================================================ - cc_table_bin (RO) Congestion control table size followed by - table entries. - - cc_settings_bin (RO) Congestion settings: port control, control - map and an array of 16 entries for the - congestion entries - increase, timer, event log - trigger threshold and the minimum injection rate - delay. - =============== ================================================ - -What: /sys/class/infiniband/qibX/ports//linkstate/loopback -What: /sys/class/infiniband/qibX/ports//linkstate/led_override -What: /sys/class/infiniband/qibX/ports//linkstate/hrtbt_enable -What: /sys/class/infiniband/qibX/ports//linkstate/status -What: /sys/class/infiniband/qibX/ports//linkstate/status_str -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - [to be documented] - - =============== =============================================== - loopback: (WO) - led_override: (WO) - hrtbt_enable: (RW) - status: (RO) - - status_str: (RO) Displays information about the link state, - possible cable/switch problems, and hardware - errors. Possible states are- "Initted", - "Present", "IB_link_up", "IB_configured" or - "Fatal_Hardware_Error". - =============== =============================================== - -What: /sys/class/infiniband/qibX/ports//diag_counters/rc_resends -What: /sys/class/infiniband/qibX/ports//diag_counters/seq_naks -What: /sys/class/infiniband/qibX/ports//diag_counters/rdma_seq -What: /sys/class/infiniband/qibX/ports//diag_counters/rnr_naks -What: /sys/class/infiniband/qibX/ports//diag_counters/other_naks -What: /sys/class/infiniband/qibX/ports//diag_counters/rc_timeouts -What: /sys/class/infiniband/qibX/ports//diag_counters/look_pkts -What: /sys/class/infiniband/qibX/ports//diag_counters/pkt_drops -What: /sys/class/infiniband/qibX/ports//diag_counters/dma_wait -What: /sys/class/infiniband/qibX/ports//diag_counters/unaligned -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - [to be documented] - - sysfs interface for Mellanox Connect-IB HCA driver mlx5 ------------------------------------------------------- From 593fd6c48861364b445f1181a7818370b8280c13 Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Mon, 7 Sep 2026 16:48:59 +0800 Subject: [PATCH 018/153] RDMA/hns: Use u32 for gid_table_len The gid_table_len is always read from HW registers or derived from u32 arithmetic, and it is never negative. Change its type from int to u32 to avoid signed/unsigned mixed-type operations with other u32 fields such as gmv_entry_num. Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260907084901.2420703-2-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_device.h | 2 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index f4f899e87ea6..4a1c2fd4c187 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -738,7 +738,7 @@ enum hns_roce_scc_algo { struct hns_roce_caps { u64 fw_ver; u8 num_ports; - int gid_table_len[HNS_ROCE_MAX_PORTS]; + u32 gid_table_len[HNS_ROCE_MAX_PORTS]; int pkey_table_len[HNS_ROCE_MAX_PORTS]; int local_ca_ack_delay; int num_uars; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 368e1d74c283..47e8b4c8bd83 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -5332,7 +5332,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, hr_reg_clear(qpc_mask, QPC_VLAN_ID); if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) { - ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n", + ibdev_err(ibdev, "sgid_index(%u) too large. max is %u\n", grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]); return -EINVAL; } From bb464eeadd301c3a53e9f92a2a472b73c42b74c4 Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Mon, 7 Sep 2026 16:49:00 +0800 Subject: [PATCH 019/153] RDMA/hns: Limit gmv_entry_num to avoid memory waste The GMV entry is a HW object corresponding to a GID. Since gid_table_len is already limited to a maximum of 256, there is no need to allocate memory for those extra GMV entries as they will never be touched. Fixes: 7243396aaf12 ("RDMA/hns: Add a max length of gid table") Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260907084901.2420703-3-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 47e8b4c8bd83..56358600c42d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2421,8 +2421,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz)); - caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / - caps->gmv_entry_sz); + caps->gmv_entry_num = caps->gid_table_len[0]; } else { u32 func_num = max_t(u32, 1, hr_dev->func_num); From 95761b8736c938759032e8d4eb42074d39d13007 Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Mon, 7 Sep 2026 16:49:01 +0800 Subject: [PATCH 020/153] RDMA/hns: Fix GID capacity loss in 64K system Our HW always handles GMV BT pages with a fixed 4K size, but the driver calculates the needed GMV BT pages number with PAGE_SIZE, which is 64K in 64K system. Only the first 4K (GMV index 0-127) can be reached by HW, causing GID capacity loss and memory waste. Split a single 64K BT page into multiple 4K pages and register them to HW per 4K block so that HW can correctly reach all GMV entries. Fixes: 32053e584e4a ("RDMA/hns: Add support for filling GMV table") Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260907084901.2420703-4-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 37 +++++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 56358600c42d..1b2504301ebe 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -4357,18 +4357,39 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj, dma_addr_t base_addr) { + u32 obj_num_per_bt = HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz; + u32 chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + PAGE_SHIFT); + u32 bt_num_per_chunk = chunk_size / HNS_HW_PAGE_SIZE; + u32 first = obj / obj_num_per_bt; + u32 last = min(first + bt_num_per_chunk, hr_dev->caps.gmv_bt_num); struct hns_roce_cmq_desc desc; - struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; - u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz); - u64 addr = to_hr_hw_page_addr(base_addr); + struct hns_roce_cmq_req *req; + u64 addr; + int ret; + u32 i; - hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false); + /* The GMV BT entry of hardware covers a fixed 4K region, so a buffer + * chunk larger than 4K must be registered to hardware with one BT + * entry per 4K block, otherwise the GMV entries beyond the first + * 4K of the chunk are unreachable. + */ + for (i = first; i < last; i++) { + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, + false); + req = (struct hns_roce_cmq_req *)desc.data; - hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); - hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); - hr_reg_write(req, CFG_GMV_BT_IDX, idx); + addr = to_hr_hw_page_addr(base_addr + + (u64)(i - first) * HNS_HW_PAGE_SIZE); + hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); + hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); + hr_reg_write(req, CFG_GMV_BT_IDX, i); - return hns_roce_cmq_send(hr_dev, &desc, 1); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + return ret; + } + + return 0; } static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, From ea42dc0936811290ac0a00475b8a37cc3379c096 Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 12:20:46 +0530 Subject: [PATCH 021/153] RDMA: fix typos in comments Fix typos in comments, reported by scripts/checkpatch.pl using the misspelling list in scripts/spelling.txt. Only touches comments, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Link: https://patch.msgid.link/20260907065047.26773-2-hemanth.selam@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/device.c | 2 +- drivers/infiniband/core/uverbs_uapi.c | 2 +- drivers/infiniband/core/verbs.c | 2 +- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 2 +- drivers/infiniband/hw/cxgb4/cq.c | 2 +- drivers/infiniband/hw/erdma/erdma_main.c | 2 +- drivers/infiniband/hw/hfi1/affinity.c | 2 +- drivers/infiniband/hw/hfi1/firmware.c | 2 +- drivers/infiniband/hw/mlx4/mcg.c | 4 ++-- drivers/infiniband/hw/mlx5/odp.c | 2 +- drivers/infiniband/hw/ocrdma/ocrdma_verbs.c | 6 +++--- drivers/infiniband/hw/qedr/verbs.c | 2 +- drivers/infiniband/hw/usnic/usnic_abi.h | 2 +- drivers/infiniband/hw/usnic/usnic_ib_verbs.c | 2 +- drivers/infiniband/hw/usnic/usnic_transport.h | 2 +- drivers/infiniband/sw/rdmavt/qp.c | 2 +- drivers/infiniband/sw/rdmavt/srq.c | 2 +- drivers/infiniband/ulp/ipoib/ipoib_vlan.c | 2 +- drivers/infiniband/ulp/iser/iscsi_iser.c | 6 +++--- drivers/infiniband/ulp/iser/iser_verbs.c | 2 +- include/rdma/ib_verbs.h | 4 ++-- include/rdma/uverbs_ioctl.h | 2 +- 22 files changed, 28 insertions(+), 28 deletions(-) diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 7a3ed5ecac00..99beb404dde0 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -893,7 +893,7 @@ static int setup_port_data(struct ib_device *device) * ib_port_immutable_read() - Read rdma port's immutable data * @dev: IB device * @port: port number whose immutable data to read. It starts with index 1 and - * valid upto including rdma_end_port(). + * valid up to including rdma_end_port(). */ const struct ib_port_immutable* ib_port_immutable_read(struct ib_device *dev, unsigned int port) diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index d150099b99d2..b67db3790f03 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -729,7 +729,7 @@ void uverbs_disassociate_api(struct uverbs_api *uapi) * * release_cleanup is the exception because * uverbs_uobject_fd_release() needs it. In this case - * the module reference held by the fops will guarentee + * the module reference held by the fops will guarantee * the type_class remains valid too. */ if (type_attrs && diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 04abc80c1327..11fd60d41f18 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -2358,7 +2358,7 @@ EXPORT_SYMBOL(ib_dereg_mr_user); * @max_num_sg: maximum sg entries available for registration. * * Notes: - * Memory registeration page/sg lists must not exceed max_num_sg. + * Memory registration page/sg lists must not exceed max_num_sg. * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed * max_num_sg * used_page_size. * diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index ccd2702db78b..ef08d42f377e 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -2903,7 +2903,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, qp->send_psn &= BTH_PSN_MASK; qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); /* DETH */ - /* Use the priviledged Q_Key for QP1 */ + /* Use the privileged Q_Key for QP1 */ qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); qp->qp1_hdr.deth.source_qpn = IB_QP1; diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index d1517f2560b9..2a7be0a1f168 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -1182,7 +1182,7 @@ void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx) struct c4iw_cq *rchp = to_c4iw_cq(qhp->ibqp.recv_cq); unsigned long flag; - /* locking heirarchy: cq lock first, then qp lock. */ + /* locking hierarchy: cq lock first, then qp lock. */ spin_lock_irqsave(&rchp->lock, flag); spin_lock(&qhp->lock); diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c index 7e87a815e853..a591d9891756 100644 --- a/drivers/infiniband/hw/erdma/erdma_main.c +++ b/drivers/infiniband/hw/erdma/erdma_main.c @@ -50,7 +50,7 @@ static int erdma_enum_and_get_netdev(struct erdma_dev *dev) struct net_device *netdev; int ret = -EPROBE_DEFER; - /* Already binded to a net_device, so we skip. */ + /* Already bound to a net_device, so we skip. */ if (dev->netdev) return 0; diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c index 2b20907c9c12..d4bf75dcd102 100644 --- a/drivers/infiniband/hw/hfi1/affinity.c +++ b/drivers/infiniband/hw/hfi1/affinity.c @@ -386,7 +386,7 @@ static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd) /* * This function creates the table for looking up CPUs for completion vectors. - * num_comp_vectors needs to have been initilized before calling this function. + * num_comp_vectors needs to have been initialized before calling this function. */ static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd, struct hfi1_affinity_node *entry) diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c index 3c228aeaaf81..a69de0a91238 100644 --- a/drivers/infiniband/hw/hfi1/firmware.c +++ b/drivers/infiniband/hw/hfi1/firmware.c @@ -77,7 +77,7 @@ struct css_header { /* size of platform configuration partition */ #define MAX_PLATFORM_CONFIG_FILE_SIZE 4096 -/* size of file of plaform configuration encoded in format version 4 */ +/* size of file of platform configuration encoded in format version 4 */ #define PLATFORM_CONFIG_FORMAT_4_FILE_SIZE 528 /* the file itself */ diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c index dc7c0daaaba5..bba178ffe1b2 100644 --- a/drivers/infiniband/hw/mlx4/mcg.c +++ b/drivers/infiniband/hw/mlx4/mcg.c @@ -433,7 +433,7 @@ static u16 cmp_rec(struct ib_sa_mcmember_data *src, } /* release group, return 1 if this was last release and group is destroyed - * timout work is canceled sync */ + * timeout work is canceled sync */ static int release_group(struct mcast_group *group, int from_timeout_handler) { struct mlx4_ib_demux_ctx *ctx = group->demux; @@ -694,7 +694,7 @@ static void mlx4_ib_mcg_work_handler(struct work_struct *work) cur_join_state = group->rec.scope_join_state & 0xf; if (method == IB_MGMT_METHOD_GET_RESP) { - /* successfull join */ + /* successful join */ if (!cur_join_state && resp_join_state) --rc; } else if (!resp_join_state) diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index b8618610737a..8dcbb0e5ccd7 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -300,7 +300,7 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, /* * Iteration one - zap the HW's MTTs. The notifiers_count ensures that * while we are doing the invalidation, no page fault will attempt to - * overwrite the same MTTs. Concurent invalidations might race us, + * overwrite the same MTTs. Concurrent invalidations might race us, * but they will write 0s as well, so no difference in the end result. */ for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index cfe3d19b73b3..97c2e18e67ad 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -1392,9 +1392,9 @@ int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, qp = get_ocrdma_qp(ibqp); dev = get_ocrdma_dev(ibqp->device); - /* syncronize with multiple context trying to change, retrive qps */ + /* synchronize with multiple context trying to change, retrieve qps */ mutex_lock(&dev->dev_lock); - /* syncronize with wqe, rqe posting and cqe processing contexts */ + /* synchronize with wqe, rqe posting and cqe processing contexts */ spin_lock_irqsave(&qp->q_lock, flags); old_qps = get_ibqp_state(qp->state); if (attr_mask & IB_QP_STATE) @@ -1599,7 +1599,7 @@ static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) */ cur_getp = cq->getp; - /* find upto when do we reap the cq. */ + /* find up to when do we reap the cq. */ stop_getp = cur_getp; do { if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 012a0ab98d6b..973d0fa7cdce 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -1075,7 +1075,7 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) /* We don't want the IRQ handler to handle a non-existing CQ so we * wait until all CNQ interrupts, if any, are received. This will always * happen and will always happen very fast. If not, then a serious error - * has occured. That is why we can use a long delay. + * has occurred. That is why we can use a long delay. * We spin for a short time so we don’t lose time on context switching * in case all the completions are handled in that span. Otherwise * we sleep for a while and check again. Since the CNQ may be diff --git a/drivers/infiniband/hw/usnic/usnic_abi.h b/drivers/infiniband/hw/usnic/usnic_abi.h index 86a82a4da0aa..a684bc54f3d4 100644 --- a/drivers/infiniband/hw/usnic/usnic_abi.h +++ b/drivers/infiniband/hw/usnic/usnic_abi.h @@ -72,7 +72,7 @@ struct usnic_ib_create_qp_resp { u64 bar_bus_addr; u32 bar_len; /* - * WQ, RQ, CQ are explicitly specified bc exposing a generic resources inteface + * WQ, RQ, CQ are explicitly specified bc exposing a generic resources interface * expands the scope of ABI to many files. */ u32 wq_cnt; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 1a1647d0e345..ebba3027cf48 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -181,7 +181,7 @@ find_free_vf_and_create_qp_grp(struct ib_qp *qp, } if (usnic_ib_share_vf) { - /* Try to find resouces on a used vf which is in pd */ + /* Try to find resources on a used vf which is in pd */ dev_list = usnic_uiom_get_dev_list(pd->umem_pd); if (IS_ERR(dev_list)) return PTR_ERR(dev_list); diff --git a/drivers/infiniband/hw/usnic/usnic_transport.h b/drivers/infiniband/hw/usnic/usnic_transport.h index 9a7a2d9755c0..c963c92218d3 100644 --- a/drivers/infiniband/hw/usnic/usnic_transport.h +++ b/drivers/infiniband/hw/usnic/usnic_transport.h @@ -50,7 +50,7 @@ int usnic_transport_sock_to_str(char *buf, int buf_sz, u16 usnic_transport_rsrv_port(enum usnic_transport_type type, u16 port_num); void usnic_transport_unrsrv_port(enum usnic_transport_type type, u16 port_num); /* - * Do a fget on the socket refered to by sock_fd and returns the socket. + * Do a fget on the socket referred to by sock_fd and returns the socket. * Socket will not be destroyed before usnic_transport_put_socket has * been called. */ diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index c40cce69e945..d8ee485c49a6 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -814,7 +814,7 @@ int rvt_alloc_rq(struct rvt_rq *rq, u32 size, int node, * * This function is called from both rvt_create_qp() and * rvt_reset_qp(). The difference is that the reset - * patch the necessary locks to protect against concurent + * patch the necessary locks to protect against concurrent * access. */ static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c index d022aa56c5bf..8122b46ed4ab 100644 --- a/drivers/infiniband/sw/rdmavt/srq.c +++ b/drivers/infiniband/sw/rdmavt/srq.c @@ -286,7 +286,7 @@ int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) } /** - * rvt_destroy_srq - destory an srq + * rvt_destroy_srq - destroy an srq * @ibsrq: srq object to destroy * @udata: user data for libibverbs.so */ diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c index ba3b45152907..697cad0d063d 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c @@ -140,7 +140,7 @@ int __ipoib_vlan_add(struct ipoib_dev_priv *ppriv, struct ipoib_dev_priv *priv, goto out_early; } - /* RTNL childs don't need proprietary sysfs entries */ + /* RTNL children don't need proprietary sysfs entries */ if (type == IPOIB_LEGACY_CHILD) { if (ipoib_cm_add_mode_attr(ndev)) goto sysfs_failed; diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c index 7df441685780..31e3a8ab6c48 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.c +++ b/drivers/infiniband/ulp/iser/iscsi_iser.c @@ -498,7 +498,7 @@ static int iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session, * iscsi_iser_conn_start() - start iscsi-iser connection * @cls_conn: iscsi class connection * - * Notes: Here iser intialize (or re-initialize) stop_completion as + * Notes: Here iser initialize (or re-initialize) stop_completion as * from this point iscsi must call conn_stop in session/connection * teardown so iser transport must wait for it. */ @@ -593,7 +593,7 @@ static inline unsigned int iser_dif_prot_caps(int prot_caps) * @qdepth: session command queue depth * @initial_cmdsn: initiator command sequnce number * - * Allocates and adds a scsi host, expose DIF supprot if + * Allocates and adds a scsi host, expose DIF support if * exists, and sets up an iscsi session. */ static struct iscsi_cls_session * @@ -866,7 +866,7 @@ static int iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) * @ep: iscsi endpoint handle * * This routine is not blocked by iser and RDMA termination process - * completion as we queue a deffered work for iser/RDMA destruction + * completion as we queue a deferred work for iser/RDMA destruction * and cleanup or actually call it immediately in case we didn't pass * iscsi conn bind/start stage, thus it is safe. */ diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index 55fe68e5b837..ab2a27d23478 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -347,7 +347,7 @@ void iser_release_work(struct work_struct *work) /* Wait for conn_stop to complete */ wait_for_completion(&iser_conn->stop_completion); - /* Wait for IB resouces cleanup to complete */ + /* Wait for IB resources cleanup to complete */ wait_for_completion(&iser_conn->ib_completion); mutex_lock(&iser_conn->state_mutex); diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index cb3b6163961b..d6eaa3fee66e 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -3532,7 +3532,7 @@ static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num) * @port_num: Port number to check * * Similar to above, but specific to iWARP connections which have a different - * managment protocol than InfiniBand. + * management protocol than InfiniBand. * * Return: true if the port supports an iWARP CM (this does not guarantee that * a CM is actually running however). @@ -3577,7 +3577,7 @@ static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num) * to this group. And it should leave the group only after all queue pairs * attached to the group have been detached. * - * Return: true if the port must undertake the additional adminstrative + * Return: true if the port must undertake the additional administrative * overhead of registering/unregistering with the SM and tracking of the * total number of queue pairs attached to the multicast group. */ diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index 80f3ba6663d0..1aaaadb8daf6 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -1142,7 +1142,7 @@ static inline int ib_is_udata_in_empty(struct ib_udata *udata) * @_udata: The system calls ib_udata struct * * Used when there is no driver response data to return. Provides forward - * compatability by zeroing any buffer the user may have provided. + * compatibility by zeroing any buffer the user may have provided. */ static inline int ib_respond_empty_udata(struct ib_udata *udata) { From 3e1de7f906ab162b23d6fe0eabccf687a98fa25f Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 12:20:47 +0530 Subject: [PATCH 022/153] RDMA: fix repeated words in comments Drop words accidentally written twice, reported by checkpatch.pl as a possible repeated word. Only touches comments, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Link: https://patch.msgid.link/20260907065047.26773-3-hemanth.selam@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/iwcm.c | 2 +- drivers/infiniband/ulp/srpt/ib_srpt.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 0b7246ec559e..4b05867dad89 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -936,7 +936,7 @@ static void cm_disconnect_handler(struct iwcm_id_private *cm_id_priv, /* * CM_ID <-- IDLE * - * If in the ESTBLISHED or CLOSING states, the QP will have have been + * If in the ESTBLISHED or CLOSING states, the QP will have been * moved by the provider to the ERR state. Disassociate the CM_ID from * the QP, move to IDLE, and remove the 'connected' reference. * diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 7197d95f2216..43c3ff1ad8e7 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -1888,7 +1888,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) * We divide up our send queue size into half SEND WRs to send the * completions, and half R/W contexts to actually do the RDMA * READ/WRITE transfers. Note that we need to allocate CQ slots for - * both both, as RDMA contexts will also post completions for the + * both, as RDMA contexts will also post completions for the * RDMA READ case. */ qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr); From a4368022a3ac5324356ae8d88e3a42d115956fd6 Mon Sep 17 00:00:00 2001 From: Nicolas Morey Date: Wed, 9 Sep 2026 18:01:31 +0200 Subject: [PATCH 023/153] RDMA/rxe: Use validated num_sge in local buffer For both SRQ and non-SRQ receive paths, the WQE is copied into a local buffer to provide a kernel-owned, validated copy. While calculating the memcpy size from the validated num_sge prevents overflow during the copy, memcpy() itself still copies num_sge from shared memory. A concurrent userspace modification before or during memcpy() leaves an unvalidated num_sge in the local buffer, leading to potential out-of-bounds reads in rxe_resp_check_length() and copy_data() causing: BUG: KASAN: slab-out-of-bounds in rxe_receiver+0x8109/0x9ec0 [rdma_rxe] Read of size 4 at addr ffff88812c4867f8 by task kworker/u9:6/361 Workqueue: rxe_wq do_work [rdma_rxe] Call Trace: rxe_receiver+0x8109/0x9ec0 [rdma_rxe] do_work+0x149/0x610 [rdma_rxe] process_one_work+0x726/0x10a0 The buggy address belongs to the object at ffff88812c486000 which belongs to the cache kmalloc-part-13-2k of size 2048 The buggy address is located 0 bytes to the right of allocated 2040-byte region [ffff88812c486000, ffff88812c4867f8) Explicitly assign the validated num_sge to the local buffer after the copy to prevent this race. Fixes: 22b8fbded65b ("RDMA/rxe: Fix TOCTOU heap overflow in get_srq_wqe") Fixes: d6ab440240a0 ("RDMA/rxe: Copy WQE to local buffer in non-SRQ receive path") Reviewed-by: Zhu Yanjun Signed-off-by: Nicolas Morey Link: https://patch.msgid.link/20260909160132.1491248-1-nmorey@suse.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_resp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index b1ea720aa34f..ba3b2d23b36b 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -288,6 +288,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) } size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); memcpy(&qp->resp.srq_wqe, wqe, size); + qp->resp.srq_wqe.wqe.dma.num_sge = num_sge; qp->resp.wqe = &qp->resp.srq_wqe.wqe; queue_advance_consumer(q, QUEUE_TYPE_FROM_CLIENT); @@ -328,6 +329,7 @@ static enum resp_states rxe_get_recv_wqe(struct rxe_qp *qp) } size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); memcpy(&qp->resp.srq_wqe, wqe, size); + qp->resp.srq_wqe.wqe.dma.num_sge = num_sge; qp->resp.wqe = &qp->resp.srq_wqe.wqe; return RESPST_CHK_LENGTH; From f7d28791433d373ecdab7b00f60ddb504e858e0a Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Fri, 11 Sep 2026 10:15:57 +0800 Subject: [PATCH 024/153] RDMA/uverbs: Make CQ handle mandatory for WQ creation UVERBS_ATTR_CREATE_WQ_CQ_HANDLE is declared UA_OPTIONAL in the ioctl method definition, so the mandatory attribute bitmap does not enforce its presence. When userspace omits it, uverbs_attr_get_obj() returns ERR_PTR(-ENOENT) and the handler stores that error pointer into wq_init_attr.cq without validation. The bogus cq pointer is then passed to the driver's create_wq callback. In the mlx5 case, create_rq() calls to_mcq(init_attr->cq) which applies container_of to the ERR_PTR value, producing a near-NULL pointer. The subsequent access in get_rq_ts_format() triggers a kernel NULL pointer dereference: BUG: kernel NULL pointer dereference, address: 0000000000000296 RIP: 0010:create_rq+0x32/0x550 [mlx5_ib] Call Trace: mlx5_ib_create_wq+0x14a/0x210 [mlx5_ib] ib_uverbs_handler_UVERBS_METHOD_WQ_CREATE+0x1f0/0x320 [ib_uverbs] ib_uverbs_run_method+0x296/0x320 [ib_uverbs] ib_uverbs_cmd_verbs+0x1a0/0x260 [ib_uverbs] ib_uverbs_ioctl+0xa8/0x120 [ib_uverbs] A WQ without a CQ was never valid; the legacy write path always required one via uobj_get_obj_read() in ib_uverbs_ex_create_wq(). Declare the attribute UA_MANDATORY so the uverbs framework rejects the ioctl early when the CQ handle is missing, before the handler ever runs. Fixes: ef3bc084a8ed ("IB/uverbs: Introduce create/destroy WQ commands over ioctl") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260911021557.2113-1-lirongqing@baidu.com Reviewed-by: Kalesh AP Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/uverbs_std_types_wq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/core/uverbs_std_types_wq.c b/drivers/infiniband/core/uverbs_std_types_wq.c index 7ded8339346f..c23f455231bf 100644 --- a/drivers/infiniband/core/uverbs_std_types_wq.c +++ b/drivers/infiniband/core/uverbs_std_types_wq.c @@ -140,7 +140,7 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_CQ_HANDLE, UVERBS_OBJECT_CQ, UVERBS_ACCESS_READ, - UA_OPTIONAL), + UA_MANDATORY), UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_WQ_EVENT_FD, UVERBS_OBJECT_ASYNC_EVENT, UVERBS_ACCESS_READ, From 8d946772ce6b33c7e55d5fb47ad4496360f92a94 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Mon, 14 Sep 2026 20:12:53 +0800 Subject: [PATCH 025/153] RDMA/mlx5: Print err code when create_qp fails Add the actual error code to the "Create QP type %d failed" log line in create_qp(), so failures can be diagnosed from dmesg alone without guessing the underlying error. Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260914121253.2193-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 0d4f8b109ad2..2f05817f583c 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -3186,7 +3186,7 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, out: if (err) { - mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type); + mlx5_ib_err(dev, "Create QP type %d failed, err = %d\n", qp->type, err); return err; } From 6334cdf33fc3c47a8e90af8ea492ad0f75c6d51d Mon Sep 17 00:00:00 2001 From: Serhat Kumral Date: Mon, 14 Sep 2026 16:05:36 +0300 Subject: [PATCH 026/153] RDMA/srpt: Fix srp_sq_size documentation Replace "Shared receive queue (SRQ)" with "Send queue". Signed-off-by: Serhat Kumral Link: https://patch.msgid.link/20260914130536.19769-1-serhatkumral1@gmail.com Reviewed-by: Bart Van Assche Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/srpt/ib_srpt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h index 4c46b301eea1..506c11b84359 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.h +++ b/drivers/infiniband/ulp/srpt/ib_srpt.h @@ -350,7 +350,7 @@ struct srpt_nexus { * struct srpt_port_attrib - attributes for SRPT port * @srp_max_rdma_size: Maximum size of SRP RDMA transfers for new connections. * @srp_max_rsp_size: Maximum size of SRP response messages in bytes. - * @srp_sq_size: Shared receive queue (SRQ) size. + * @srp_sq_size: Send queue size. * @use_srq: Whether or not to use SRQ. */ struct srpt_port_attrib { From 18922a5387b65d023737f6443919cec66755e64a Mon Sep 17 00:00:00 2001 From: Long Li Date: Mon, 14 Sep 2026 21:31:21 -0700 Subject: [PATCH 027/153] MAINTAINERS: hand over MANA RDMA driver to Wei Hu Wei Hu takes over maintainership of the Microsoft MANA RDMA driver. Signed-off-by: Long Li Link: https://patch.msgid.link/20260915043121.10370-1-longli@kernel.org Signed-off-by: Leon Romanovsky --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 3a19da74d00c..9308af18bc3d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18051,7 +18051,7 @@ F: include/linux/cciss*.h F: include/uapi/linux/cciss*.h MICROSOFT MANA RDMA DRIVER -M: Long Li +M: Wei Hu M: Konstantin Taranov L: linux-rdma@vger.kernel.org L: linux-hyperv@vger.kernel.org From 653c5b4605f65355d3b3f1059784e56c77f146b3 Mon Sep 17 00:00:00 2001 From: Junxian Huang Date: Mon, 14 Sep 2026 10:48:00 +0800 Subject: [PATCH 028/153] RDMA/hns: Fix the use of uninitialized variable The variable 'i' is used without being initialized in active-backup mode. Fixes: d9023e461b73 ("RDMA/hns: Implement bonding init/uninit process") Signed-off-by: Junxian Huang Link: https://patch.msgid.link/20260914024800.132429-1-huangjunxian6@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_bond.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index f2b12ae13a58..8d8272705fb7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -81,7 +81,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp, { struct net_device *active_dev; struct net_device *old_dev; - int i, ret = 0; + int i = 0, ret = 0; if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) { rcu_read_lock(); @@ -89,7 +89,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp, bond_option_active_slave_get_rcu(netdev_priv(bond_grp->upper_dev)); rcu_read_unlock(); } else { - for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { + for (; i < ROCE_BOND_FUNC_MAX; i++) { active_dev = bond_grp->bond_func_info[i].net_dev; if (active_dev && ib_get_curr_port_state(active_dev) == IB_PORT_ACTIVE) From fad7b0dcb0821c0f201df3d85065145857a6f55b Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 16 Sep 2026 14:44:35 +0800 Subject: [PATCH 029/153] RDMA/core: Fix swapped list_add_tail() arguments in ib_add_sub_device() ib_add_sub_device() links a new sub-device into its parent's sub-device list with: list_add_tail(&parent->subdev_list_head, &sub->subdev_list); list_add_tail(new, head) expects the node to insert as the first argument and the list head as the second, so the call above does the opposite of what was intended: it treats the parent's list head as the new node and the sub-device's node as the list head. Because _ib_alloc_device() initialises subdev_list to a self-referencing empty head, the misplaced insertion does not crash, but it corrupts the parent->subdev_list_head list. After adding two or more sub-devices, only the last one is reachable through the parent's head, and ib_del_sub_device_and_put()'s list_del() on &sub->subdev_list rewrites the head pointers, potentially severing earlier sub-devices from the chain. During parent teardown, the reverse iteration over subdev_list_head then skips the orphaned sub-devices, so their del_sub_dev() callbacks are never invoked and ib_device_put() on the parent is never paired, leaking both the HW sub-devices and a refcount. Swap the arguments to match the documented intent: insert the sub-device node into the parent's list head. Fixes: bca51197620a ("RDMA/core: Support IB sub device with type "SMI"") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260916064435.2180-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 99beb404dde0..29af9fe860e2 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -3067,7 +3067,7 @@ int ib_add_sub_device(struct ib_device *parent, sub->parent = parent; mutex_lock(&parent->subdev_lock); - list_add_tail(&parent->subdev_list_head, &sub->subdev_list); + list_add_tail(&sub->subdev_list, &parent->subdev_list_head); mutex_unlock(&parent->subdev_lock); return ret; From 391c5d826fd0a5dacad54613dbef74c50e966f5c Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 16 Sep 2026 15:30:42 +0800 Subject: [PATCH 030/153] RDMA/restrack: Don't set RESTRACK_DD mark after a failed xa_insert() When adding a QP to the restrack xarray, rdma_restrack_add() does: ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL); if (ret) res->id = 0; if (qp->qp_type >= IB_QPT_DRIVER) xa_set_mark(&rt->xa, res->id, RESTRACK_DD); The xa_set_mark() call is not guarded by "!ret". When xa_insert() fails (possible on a qp_num collision or memory pressure), res->id is reset to 0, yet the mark is still applied to index 0. If a legitimate QP whose qp_num is 0 already occupies that slot - e.g. a normal QP or the SMI QP - it is spuriously tagged as driver-private (RESTRACK_DD), and the netlink dump path res_get_common_dumpit() then hides it from "rdma res" output whenever the caller does not request driver details. Only set the mark when the insertion actually succeeded by folding the mark into the non-failure branch. Fixes: e18fa0bbcedf8 ("RDMA/core: Add an option to display driver-specific QPs in the rdmatool") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260916073042.1746-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/restrack.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index f89a81dad72f..a8b838fdc7b5 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -250,8 +250,7 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL); if (ret) res->id = 0; - - if (qp->qp_type >= IB_QPT_DRIVER) + else if (qp->qp_type >= IB_QPT_DRIVER) xa_set_mark(&rt->xa, res->id, RESTRACK_DD); } else if (res->type == RDMA_RESTRACK_COUNTER) { /* Special case to ensure that cntn points to right counter */ From 9dcacdc41083cccff74292851bf9fde999e0731a Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 16 Sep 2026 16:55:29 +0800 Subject: [PATCH 031/153] RDMA/uverbs: Drop restrack ref on ib_init_ucontext() failure in GET_CONTEXT The UVERBS_METHOD_GET_CONTEXT handler allocates the ucontext via ib_alloc_ucontext(), which calls rdma_restrack_new() (initialising the kref to 1) and rdma_restrack_set_name(NULL), the latter attaching the current task and taking a task_struct reference. When ib_init_ucontext() subsequently fails, the error path only does: kfree(attrs->context); attrs->context = NULL; without first calling rdma_restrack_put() on &attrs->context->res. The kref therefore never reaches zero, restrack_release() is never invoked, and put_task_struct() is never called, so the task_struct reference taken during set_name is leaked permanently. A local user with access to an RDMA device can repeat this path (e.g. by hitting an RDMA cgroup limit or supplying invalid ucaps) and leak one task_struct reference per attempt, eventually preventing those processes from being reaped or exhausting memory. The legacy ib_uverbs_get_context() path already handles this correctly at its err_ucontext label, where rdma_restrack_put() precedes kfree(). Mirror that ordering here. Fixes: a1123418ba10 ("RDMA/uverbs: Add ioctl command to get a device context") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260916085529.1838-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/uverbs_std_types_device.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index ce0a7de00405..db65032e80b4 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -272,6 +272,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( return ret; ret = ib_init_ucontext(attrs); if (ret) { + rdma_restrack_put(&attrs->context->res); kfree(attrs->context); attrs->context = NULL; return ret; From b63d21631f836528ef660d7b5a0d84890ff6dbd5 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 16 Sep 2026 20:09:26 +0800 Subject: [PATCH 032/153] RDMA/core: Fix partial copy of IPv6 flow_lbl in LAG hash skb rdma_build_skb() constructs a synthetic IPv6 header for LAG slave selection and copies the flow label from the AH attribute: memcpy(&ip6h->flow_lbl, &ah_attr->grh.flow_label, sizeof(*ip6h->flow_lbl)); ipv6hdr.flow_lbl is __u8[3], so sizeof(*ip6h->flow_lbl) dereferences the array to a single __u8 and yields 1. The memcpy therefore copies only the first byte of the flow label, leaving flow_lbl[1] and flow_lbl[2] uninitialised in the skb buffer (alloc_skb() does not zero the data). The resulting LAG hash is computed over one byte of real flow-label data and two bytes of kmalloc residue, so IPv6 RoCEv2 traffic on a bonded interface can be steered to the wrong slave. Use sizeof(ip6h->flow_lbl) (the array, 3 bytes) so the whole flow label is copied. Fixes: bd3920eac1331 ("RDMA/core: Add LAG functionality") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260916120926.1761-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/lag.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c index 00fe241737ff..88aac911e8ee 100644 --- a/drivers/infiniband/core/lag.c +++ b/drivers/infiniband/core/lag.c @@ -59,7 +59,7 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev, ip6h->version = 6; ip6h->nexthdr = IPPROTO_UDP; memcpy(&ip6h->flow_lbl, &ah_attr->grh.flow_label, - sizeof(*ip6h->flow_lbl)); + sizeof(ip6h->flow_lbl)); memcpy(&ip6h->saddr, ah_attr->grh.sgid_attr->gid.raw, sizeof(struct in6_addr)); memcpy(&ip6h->daddr, ah_attr->grh.dgid.raw, From 8a8088b899e4cdfe558428cd6113d45df81625be Mon Sep 17 00:00:00 2001 From: Guangshuo Li Date: Mon, 14 Sep 2026 20:28:40 +0800 Subject: [PATCH 033/153] RDMA/irdma: avoid use-after-free in icrdma_remove icrdma_remove() dereferences iwdev after calling irdma_ib_unregister_device(): irdma_ib_unregister_device(iwdev) -> ib_unregister_device(&iwdev->ibdev) -> __ib_unregister_device() -> ib_dealloc_device() irdma registers irdma_ib_dealloc_device() as the dealloc_driver callback in its ib_device_ops. The RDMA core explicitly documents ib_unregister_device() that when ops.dealloc_driver is used, ib_dev will be freed upon return from the function. The ib_device is embedded in struct irdma_device and iwdev itself was allocated by ib_alloc_device(irdma_device, ibdev). Consequently, once irdma_ib_unregister_device() returns, iwdev must no longer be dereferenced. However, icrdma_remove() currently accesses iwdev->rf afterwards when deinitializing interrupts, destroying ah_tbl_lock, and freeing rf. Although the final ib_device storage is released with kfree_rcu(), the object lifetime has already ended and callers must not rely on the RCU grace period to continue dereferencing iwdev. Save iwdev->rf before unregistering the RDMA device and use the saved pointer for the remaining cleanup. This issue was found by manual code inspection. Fixes: 8498a30e1b94 ("RDMA/irdma: Register auxiliary driver and implement private channel OPs") Cc: stable@vger.kernel.org Signed-off-by: Guangshuo Li Link: https://patch.msgid.link/20260914122840.1709743-1-lgs201920130244@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/icrdma_if.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/irdma/icrdma_if.c b/drivers/infiniband/hw/irdma/icrdma_if.c index 4b451d8482a4..f405ef761f84 100644 --- a/drivers/infiniband/hw/irdma/icrdma_if.c +++ b/drivers/infiniband/hw/irdma/icrdma_if.c @@ -318,14 +318,15 @@ static void icrdma_remove(struct auxiliary_device *aux_dev) container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev); struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info; struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev); - u8 rdma_ver = iwdev->rf->rdma_ver; + struct irdma_pci_f *rf = iwdev->rf; + u8 rdma_ver = rf->rdma_ver; ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false); irdma_ib_unregister_device(iwdev); - icrdma_deinit_interrupts(iwdev->rf, cdev_info); - mutex_destroy(&iwdev->rf->ah_tbl_lock); + icrdma_deinit_interrupts(rf, cdev_info); + mutex_destroy(&rf->ah_tbl_lock); - kfree(iwdev->rf); + kfree(rf); pr_debug("INIT: Gen[%d] func[%d] device remove success\n", rdma_ver, PCI_FUNC(cdev_info->pdev->devfn)); From c6aa5feb52106664df44ef3c8c1da48f703ef8f0 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 18:33:28 +0300 Subject: [PATCH 034/153] RDMA/mlx5: Use unsigned comparison in the CQ cleanup loop __mlx5_ib_cq_clean() sweeps the CQ backwards from the producer index down to the consumer index: while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) Both indexes are free running u32 counters, so the comparison has to be done modulo 2^32. Casting each operand to int and subtracting does not do that: the subtraction overflows whenever the two indexes straddle 2^31, which is undefined behaviour, and a compiler that assumes signed overflow cannot occur is free to discard the subtraction and fold the expression into a plain signed comparison. That comparison is not wraparound safe. The kernel is built with -fno-strict-overflow, so gcc and clang both retain the subtraction today and the generated code is unaffected; there is no known user-visible impact from the current code. Still, correctness here shouldn't depend on that build flag. Replace the loop with a plain unsigned equality check instead. while (prod_index != cq->mcq.cons_index) { --prod_index; ... The preceding forward scan starts prod_index at cons_index and stops no later than cons_index + cq->ibcq.cqe, so the two indexes are at most cq->ibcq.cqe apart. Decrementing prod_index reaches cons_index in exactly that many iterations regardless of whether either counter has wrapped, because the loop no longer compares magnitudes at all. Signed-off-by: Yishai Hadas Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260915-fix-cq-cleanup-v1-1-e991944cf898@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/cq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 49b4bf148a4a..51892911e32a 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -1169,7 +1169,8 @@ void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq) /* Now sweep backwards through the CQ, removing CQ entries * that match our QP by copying older entries on top of them. */ - while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { + while (prod_index != cq->mcq.cons_index) { + --prod_index; cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; if (is_equal_rsn(cqe64, rsn)) { From c5fe428bb23edc833ebcf99de0ea92c1625a3c7b Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 18:33:29 +0300 Subject: [PATCH 035/153] RDMA/mlx4: Use unsigned comparison in the CQ cleanup loop __mlx4_ib_cq_clean() sweeps the CQ backwards from the producer index down to the consumer index: while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) Both indexes are free running u32 counters, so the comparison has to be done modulo 2^32. Casting each operand to int and subtracting does not do that: the subtraction overflows whenever the two indexes straddle 2^31, which is undefined behaviour, and a compiler that assumes signed overflow cannot occur is free to discard the subtraction and fold the expression into a plain signed comparison. That comparison is not wraparound safe. The kernel is built with -fno-strict-overflow, so gcc and clang both retain the subtraction today and the generated code is unaffected; there is no known user-visible impact from the current code. Still, correctness here shouldn't depend on that build flag. Replace the loop with a plain unsigned equality check instead: while (prod_index != cq->mcq.cons_index) { --prod_index; ... The preceding forward scan starts prod_index at cons_index and stops no later than cons_index + cq->ibcq.cqe, so the two indexes are at most cq->ibcq.cqe apart. Decrementing prod_index reaches cons_index in exactly that many iterations regardless of whether either counter has wrapped, because the loop no longer compares magnitudes at all. Signed-off-by: Yishai Hadas Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260915-fix-cq-cleanup-v1-2-e991944cf898@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx4/cq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 887912469742..98e3eb5cf96e 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -989,7 +989,8 @@ void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) * Now sweep backwards through the CQ, removing CQ entries * that match our QP by copying older entries on top of them. */ - while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { + while (prod_index != cq->mcq.cons_index) { + --prod_index; cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); cqe += cqe_inc; From b934024d3aafb9de938a6eb424bbb485e2053450 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 18:33:30 +0300 Subject: [PATCH 036/153] RDMA/mthca: Use unsigned comparison in the CQ cleanup loop mthca_cq_clean() sweeps the CQ backwards from the producer index down to the consumer index: while ((int) --prod_index - (int) cq->cons_index >= 0) Both indexes are free running u32 counters, so the comparison has to be done modulo 2^32. Casting each operand to int and subtracting does not do that: the subtraction overflows whenever the two indexes straddle 2^31, which is undefined behaviour, and a compiler that assumes signed overflow cannot occur is free to discard the subtraction and fold the expression into a plain signed comparison. That comparison is not wraparound safe. The kernel is built with -fno-strict-overflow, so gcc and clang both retain the subtraction today and the generated code is unaffected; there is no known user-visible impact from the current code. Still, correctness here shouldn't depend on that build flag. Replace the loop with a plain unsigned equality check instead: while (prod_index != cq->cons_index) { --prod_index; ... The preceding forward scan starts prod_index at cons_index and stops no later than cons_index + cq->ibcq.cqe, so the two indexes are at most cq->ibcq.cqe apart. Decrementing prod_index reaches cons_index in exactly that many iterations regardless of whether either counter has wrapped, because the loop no longer compares magnitudes at all. Signed-off-by: Yishai Hadas Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260915-fix-cq-cleanup-v1-3-e991944cf898@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mthca/mthca_cq.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c index 26c3408dcaca..39d4ff15849e 100644 --- a/drivers/infiniband/hw/mthca/mthca_cq.c +++ b/drivers/infiniband/hw/mthca/mthca_cq.c @@ -300,7 +300,8 @@ void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, * Now sweep backwards through the CQ, removing CQ entries * that match our QP by copying older entries on top of them. */ - while ((int) --prod_index - (int) cq->cons_index >= 0) { + while (prod_index != cq->cons_index) { + --prod_index; cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); if (cqe->my_qpn == cpu_to_be32(qpn)) { if (srq && is_recv_cqe(cqe)) From ebb65190349537933a796ee662d05d0464bd4066 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 18:33:31 +0300 Subject: [PATCH 037/153] RDMA/hns: Use unsigned comparison in the CQ cleanup loop __hns_roce_v2_cq_clean() sweeps the CQ backwards from the producer index down to the consumer index: while ((int) --prod_index - (int) hr_cq->cons_index >= 0) Both indexes are free running u32 counters, so the comparison has to be done modulo 2^32. Casting each operand to int and subtracting does not do that: the subtraction overflows whenever the two indexes straddle 2^31, which is undefined behaviour, and a compiler that assumes signed overflow cannot occur is free to discard the subtraction and fold the expression into a plain signed comparison. That comparison is not wraparound safe. The kernel is built with -fno-strict-overflow, so gcc and clang both retain the subtraction today and the generated code is unaffected; there is no known user-visible impact from the current code. Still, correctness here shouldn't depend on that build flag. Replace the loop with a plain unsigned equality check instead: while (prod_index != hr_cq->cons_index) { --prod_index; ... The preceding forward scan starts prod_index at cons_index and advances it only while get_sw_cqe_v2() still reports a software owned entry, so the two indexes stay within one CQ's worth of entries. Decrementing prod_index reaches cons_index in exactly that many iterations regardless of whether either counter has wrapped, because the loop no longer compares magnitudes at all. Signed-off-by: Yishai Hadas Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20260915-fix-cq-cleanup-v1-4-e991944cf898@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 1b2504301ebe..35be5b038f68 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -3828,7 +3828,8 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, * Now backwards through the CQ, removing CQ entries * that match our QP by overwriting them with next entries. */ - while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { + while (prod_index != hr_cq->cons_index) { + --prod_index; cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe); if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) { if (srq && hr_reg_read(cqe, CQE_S_R)) { From ba88a28b47a703c3637ead33b1c70afab0803513 Mon Sep 17 00:00:00 2001 From: Ammar Ratnani Date: Tue, 8 Sep 2026 08:44:26 -0700 Subject: [PATCH 038/153] RDMA/core: Force disconnect if DREP and DREQ fail When we call `rdma_disconnect`, it's possible that we fail to send both a DREQ and a DREP. In that case, we currently either do nothing or move into the `IB_CM_TIMEWAIT` state. Either way, we remain in the `RDMA_CM_CONNECT` state. Anyone waiting for us to disconnect hangs. Instead, while we're connected, when we call `rdma_disconnect`, and fail to send both a DREQ and a DREP, mark the connection as disconnected. Call the upper layer's handler for the disconnect, and destroy the connection if instructed. Cc: Leon Romanovsky Cc: Jason Gunthorpe Cc: Namjae Jeon Cc: Paulo Alcantara Cc: Stefan Metzmacher Cc: Tom Talpey Cc: linux-rdma@vger.kernel.org Cc: linux-cifs@vger.kernel.org Cc: samba-technical@lists.samba.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Ammar Ratnani Link: https://patch.msgid.link/20260908154454.10966-2-ammrat13@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/cma.c | 34 +++++++++++++++++++++++++---- drivers/infiniband/core/cma_trace.h | 1 + 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 73170b15fc3d..337a49d1acf7 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -4823,6 +4823,29 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data, } EXPORT_SYMBOL(rdma_reject); +static int cma_force_disconnect(struct rdma_id_private *id_priv) +{ + struct cma_work *work; + + work = kzalloc_obj(*work); + if (!work) + return -ENOMEM; + + INIT_WORK(&work->work, cma_work_handler); + work->old_state = RDMA_CM_CONNECT; + work->new_state = RDMA_CM_DISCONNECT; + work->event.event = RDMA_CM_EVENT_DISCONNECTED; + work->event.status = -ECONNABORTED; + + cma_id_get(id_priv); + work->id = id_priv; + + trace_cm_force_disconnect(id_priv); + + queue_work(cma_wq, &work->work); + return 0; +} + int rdma_disconnect(struct rdma_cm_id *id) { struct rdma_id_private *id_priv; @@ -4838,12 +4861,15 @@ int rdma_disconnect(struct rdma_cm_id *id) goto out; /* Initiate or respond to a disconnect. */ trace_cm_disconnect(id_priv); - if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) { - if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) - trace_cm_sent_drep(id_priv); - } else { + if (!ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) { trace_cm_sent_dreq(id_priv); + goto out; } + if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) { + trace_cm_sent_drep(id_priv); + goto out; + } + ret = cma_force_disconnect(id_priv); } else if (rdma_cap_iw_cm(id->device, id->port_num)) { ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); } else diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h index 3456d5f3aa47..3d05d8e80856 100644 --- a/drivers/infiniband/core/cma_trace.h +++ b/drivers/infiniband/core/cma_trace.h @@ -61,6 +61,7 @@ DEFINE_CMA_FSM_EVENT(send_sidr_rep); DEFINE_CMA_FSM_EVENT(disconnect); DEFINE_CMA_FSM_EVENT(sent_drep); DEFINE_CMA_FSM_EVENT(sent_dreq); +DEFINE_CMA_FSM_EVENT(force_disconnect); DEFINE_CMA_FSM_EVENT(id_destroy); TRACE_EVENT(cm_id_attach, From 5bb710151753fe53047b1d34b1b54d0f179cec68 Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Tue, 15 Sep 2026 17:03:17 +0000 Subject: [PATCH 039/153] RDMA/irdma: Fix erroneous -ENOMEM for large MR registration Commit cc8997c94bf3 ("RDMA/irdma: Refactor PBLE functions") replaced the explicit level1_only boolean with a bit mask but did not properly convert the "if (level1_only)" check in irdma_get_pble() to be "lvl == PBLE_LEVEL_1" and instead left it as "if (lvl)". User MR registration passes "lvl = PBLE_LEVEL_1 | PBLE_LEVEL_2" so this check always evaluates to true and causes the loop to bail after grabbing a single SD even if the region actually requires more than one SD (as is the case for regions larger than 1 GiB with 4k pages). As a result, these registrations can fail with -ENOMEM even if there are sufficient PBLE resources. Fixes: cc8997c94bf3 ("RDMA/irdma: Refactor PBLE functions") Signed-off-by: Jacob Moroni Link: https://patch.msgid.link/20260915170317.2901672-1-jmoroni@google.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/pble.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c index 28dfad7f940c..14318af97d58 100644 --- a/drivers/infiniband/hw/irdma/pble.c +++ b/drivers/infiniband/hw/irdma/pble.c @@ -482,7 +482,7 @@ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl); /* if level1_only, only go through it once */ - if (!status || lvl) + if (!status || lvl == PBLE_LEVEL_1) break; } From 9c40f13a63a4b87177de3e8c318af25068fc6ab4 Mon Sep 17 00:00:00 2001 From: Jeremy Bongio Date: Wed, 16 Sep 2026 17:32:03 +0000 Subject: [PATCH 040/153] RDMA/irdma: Use kvzalloc for lvl2 leafmem allocation During VM live migration or when registering large memory regions (e.g., guest memory >= 256 GB backed by 4 KiB pages), get_lvl2_pble() calculates total leaves exceeding ~131,072. The memory required for lvl2->leafmem.va exceeds 4 MiB (sizeof(*leaf) * total > 5.24 MiB). Because kzalloc() requires physically contiguous pages, requests larger than MAX_PAGE_ORDER (4 MiB on x86_64) fail and trigger a warning: WARNING: CPU: 123 PID: 119906 at mm/page_alloc.c:5997 __alloc_frozen_pages_noprof+0x50c/0x1050 Call Trace: alloc_frozen_pages_noprof+0xc0/0x240 ___kmalloc_large_node+0x3d/0xd0 __kmalloc_large_node_noprof+0x1b/0x90 __kmalloc_noprof+0x254/0x660 get_lvl1_lvl2_pble+0xbc/0x270 [irdma] irdma_get_pble+0x44/0xc0 [irdma] irdma_setup_pbles+0x46/0x1d0 [irdma] irdma_reg_user_mr_type_mem+0x41/0x230 [irdma] irdma_reg_user_mr+0x194/0x1c0 [irdma] ib_uverbs_reg_mr+0x1b0/0x2c0 [ib_uverbs] The leafmem array is purely host software tracking structures (struct irdma_pble_info) and does not require physically contiguous memory. Switch the allocation from kzalloc() to kvzalloc() and free it with kvfree() so that requests exceeding MAX_PAGE_ORDER fall back to virtually contiguous memory via vmalloc. Fixes: e8c4dbc2fcac ("RDMA/irdma: Add PBLE resource manager") Assisted-by: LLM Signed-off-by: Jeremy Bongio Link: https://patch.msgid.link/20260916173203.2555458-1-jbongio@google.com Tested-by: Jacob Moroni Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/pble.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c index 14318af97d58..0c2c5e96a05a 100644 --- a/drivers/infiniband/hw/irdma/pble.c +++ b/drivers/infiniband/hw/irdma/pble.c @@ -330,7 +330,7 @@ static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc, if (root->addr) irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo); - kfree(lvl2->leafmem.va); + kvfree(lvl2->leafmem.va); lvl2->leaf = NULL; } @@ -358,7 +358,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, lvl2->leaf_cnt = total; lvl2->leafmem.size = (sizeof(*leaf) * total); - lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL); + lvl2->leafmem.va = kvzalloc(lvl2->leafmem.size, GFP_KERNEL); if (!lvl2->leafmem.va) return -ENOMEM; @@ -367,7 +367,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo, total << 3, &root->addr, &fpm_addr); if (ret_code) { - kfree(lvl2->leafmem.va); + kvfree(lvl2->leafmem.va); lvl2->leaf = NULL; return -ENOMEM; } From 2a770e6a64d66fc3a7317669ecca9773d8339ead Mon Sep 17 00:00:00 2001 From: Slavin Liu Date: Fri, 11 Sep 2026 14:09:09 +0800 Subject: [PATCH 041/153] RDMA/irdma: check vport device allocation An existing RDMA function does not guarantee allocation of the new vport device. Return -ENOMEM before initializing a NULL iwdev. Detected by static analysis and reviewed with AI-assisted source auditing. Fixes: 2ad49ae7330b ("RDMA/irdma: Introduce GEN3 vPort driver support") Assisted-by: LLM Signed-off-by: Slavin Liu Link: https://patch.msgid.link/20260911060909.94219-1-bolin.liu@seu.edu.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/main.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c index 95957d52883d..106915b51e7c 100644 --- a/drivers/infiniband/hw/irdma/main.c +++ b/drivers/infiniband/hw/irdma/main.c @@ -80,6 +80,8 @@ static int ig3rdma_vport_probe(struct auxiliary_device *aux_dev, return -ENOMEM; } iwdev = ib_alloc_device(irdma_device, ibdev); + if (!iwdev) + return -ENOMEM; /* Fill iwdev info */ iwdev->is_vport = true; iwdev->rf = rf; From 5fb3051aa4611ae18e304814fb1b74a30065fd85 Mon Sep 17 00:00:00 2001 From: Wentao Liang Date: Wed, 16 Sep 2026 18:35:30 +0000 Subject: [PATCH 042/153] RDMA/cxgb4: Fix neigh reference leak in rx_pkt() If ip_dev_find() fails on the loopback path the code jumps to free_dst without releasing the neighbour reference taken by dst_neigh_lookup_skb(), leaking one neigh reference. Release the neighbour before jumping to free_dst. Fixes: ef42520240aa ("RDMA/cxgb4: add null-ptr-check after ip_dev_find()") Signed-off-by: Wentao Liang Link: https://patch.msgid.link/20260916183530.2092534-1-vulab@iscas.ac.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/cxgb4/cm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index b3b45c49077d..f57a31cf4fc7 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -4156,6 +4156,7 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb) pdev = ip_dev_find(&init_net, iph->daddr); if (!pdev) { pr_err("%s - failed to find device!\n", __func__); + neigh_release(neigh); goto free_dst; } e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, From 5eb6ecb612841927b5ede554a0e9d88240934282 Mon Sep 17 00:00:00 2001 From: Wentao Liang Date: Wed, 16 Sep 2026 18:37:15 +0000 Subject: [PATCH 043/153] RDMA/cxgb4: Fix skb reference leak in rx_pkt() rx_pkt() takes an extra reference on the received skb so it can be handed to the firmware completion path via req->cookie. When the work request skb allocation in send_fw_pass_open_req() fails the function returns without releasing that extra reference, leaking one skb reference. Free the packet skb before returning when the allocation fails, matching the cxgb4_ofld_send() failure path. Fixes: 1cab775c3e75 ("RDMA/cxgb4: Fix LE hash collision bug for passive open connection") Signed-off-by: Wentao Liang Link: https://patch.msgid.link/20260916183715.2092663-1-vulab@iscas.ac.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/cxgb4/cm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index f57a31cf4fc7..a6f520f49d59 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -4006,8 +4006,10 @@ static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb, int ret; req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL); - if (!req_skb) + if (!req_skb) { + kfree_skb(skb); return; + } req = __skb_put_zero(req_skb, sizeof(*req)); req->op_compl = htonl(WR_OP_V(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL_F); req->len16_pkd = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16))); From 65f3eab431944415bbe5e3af1bbb2736d4bb1523 Mon Sep 17 00:00:00 2001 From: Wentao Liang Date: Wed, 16 Sep 2026 18:39:21 +0000 Subject: [PATCH 044/153] RDMA/mlx5: Fix RQ resource reference leak in mlx5_ib_wq_event() Since commit 312b8f79eb05 ("RDMA/mlx: Calling qp event handler in workqueue context") rsc_event_notifier() expects the event handler to drop the resource reference taken by mlx5_get_rsc(). mlx5_ib_wq_event() was missed by that conversion, so every RQ event leaks one reference, which keeps destroy_resource_common() waiting on the completion. Put the resource reference on all exits of the handler. Fixes: 312b8f79eb05 ("RDMA/mlx: Calling qp event handler in workqueue context") Signed-off-by: Wentao Liang Link: https://patch.msgid.link/20260916183921.2092821-1-vulab@iscas.ac.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 2f05817f583c..b1bda6e5458b 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -5344,11 +5344,14 @@ static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type) break; default: mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn); - return; + goto out; } rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context); } + +out: + mlx5_core_res_put(&core_qp->common); } static int set_delay_drop(struct mlx5_ib_dev *dev) From ef15197f81ffedf29560edf2221199d92fd83dc2 Mon Sep 17 00:00:00 2001 From: Wentao Liang Date: Wed, 16 Sep 2026 18:44:00 +0000 Subject: [PATCH 045/153] RDMA/siw: Fix qp use-after-free in siw_get_base_qp() The reference taken by siw_qp_id2obj() is dropped before the address of the embedded ib_qp is taken from the QP. If that reference was the last one the QP is freed and the pointer returned to the iwarp core is dangling; the core only pins the QP with iw_add_ref() after the call has returned. Copy the pointer while the reference is still held. Fixes: 6c52fdc244b5 ("rdma/siw: connection management") Signed-off-by: Wentao Liang Link: https://patch.msgid.link/20260916184400.2093106-1-vulab@iscas.ac.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/siw/siw_main.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c index 9735b75ac933..c08796e4262a 100644 --- a/drivers/infiniband/sw/siw/siw_main.c +++ b/drivers/infiniband/sw/siw/siw_main.c @@ -217,8 +217,11 @@ static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id) /* * siw_qp_id2obj() increments object reference count */ + struct ib_qp *base_qp = &qp->base_qp; + siw_qp_put(qp); - return &qp->base_qp; + + return base_qp; } return NULL; } From d1fa82ef992bc18850d27bc0bdc7a5cd0821d7b3 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Thu, 17 Sep 2026 13:20:12 +0800 Subject: [PATCH 046/153] RDMA/multicast: Fix rb-tree corruption on MGID reassignment in join_handler When the SA assigns a wildcard (mgid0) join a concrete MGID that already has a group in the port table, join_handler() erases the group from the tree and calls mcast_insert() with allow_duplicates=false. mcast_insert() then finds the existing group and returns it without re-inserting, but join_handler() discards the return value, leaving the group erased yet still alive. When the orphaned group's refcount later drops to zero, release_group() calls rb_erase() on the node that is no longer in the tree, corrupting the rb-tree (a stale parent pointer causes a valid subtree link to be overwritten with NULL). Clear the node with RB_CLEAR_NODE() after the erase in join_handler() so release_group() can detect via RB_EMPTY_NODE() that it is already out of the tree and skip the second erase. A successful re-insert by mcast_insert() restores the in-tree state via rb_link_node(). Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260917052012.2181-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/multicast.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c index bea7df3dd8f3..f549fd928c1c 100644 --- a/drivers/infiniband/core/multicast.c +++ b/drivers/infiniband/core/multicast.c @@ -189,7 +189,10 @@ static void release_group(struct mcast_group *group) spin_lock_irqsave(&port->lock, flags); if (atomic_dec_and_test(&group->refcount)) { - rb_erase(&group->node, &port->table); + + if (!RB_EMPTY_NODE(&group->node)) + rb_erase(&group->node, &port->table); + spin_unlock_irqrestore(&port->lock, flags); kfree(group); deref_port(port); @@ -533,6 +536,8 @@ static void join_handler(int status, struct ib_sa_mcmember_rec *rec, group->rec = *rec; if (mgids_changed) { rb_erase(&group->node, &group->port->table); + + RB_CLEAR_NODE(&group->node); is_mgid0 = !memcmp(&mgid0, &group->rec.mgid, sizeof(mgid0)); mcast_insert(group->port, group, is_mgid0); From 193aaf5b57f05d9f0403cfdaff3eda3b8d11ec13 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Thu, 17 Sep 2026 16:54:19 +0800 Subject: [PATCH 047/153] RDMA/mlx5: Guard steering anchor resource destruction on create error The steering anchor resources are shared between anchors with the same priority and are protected by rule_goto_table_ref. When steering anchor creation succeeds but uverbs_copy_to() fails, the error path drops the reference acquired for the new anchor. However, it unconditionally calls mlx5_steering_anchor_destroy_res(), even when other anchors are still using the shared resources. This can destroy the shared flow table, flow groups, and rules while existing anchors still hold references to them, resulting in a use-after-free when those anchors are subsequently cleaned up. Only destroy the shared resources when dropping the reference makes rule_goto_table_ref zero, consistent with the existing cleanup path. Fixes: e1f4a52ac171 ("RDMA/mlx5: Create an indirect flow table for steering anchor") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260917085419.1792-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/fs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c index b155baee0017..98cdc5c57155 100644 --- a/drivers/infiniband/hw/mlx5/fs.c +++ b/drivers/infiniband/hw/mlx5/fs.c @@ -3056,8 +3056,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( return 0; destroy_res: - --ft_prio->anchor.rule_goto_table_ref; - mlx5_steering_anchor_destroy_res(ft_prio); + if (!--ft_prio->anchor.rule_goto_table_ref) + mlx5_steering_anchor_destroy_res(ft_prio); put_flow_table: put_flow_table(dev, ft_prio, true); free_obj: From 9a413ed37d54b305666844d17fc49b8f3773c9d2 Mon Sep 17 00:00:00 2001 From: Cheng Xu Date: Thu, 17 Sep 2026 20:04:55 +0800 Subject: [PATCH 048/153] RDMA/erdma: Unwind kernel QP initialization failures Allocation failures currently use the normal kernel QP teardown path. Unwind failures directly so free_kernel_qp() only handles complete QPs. Signed-off-by: Cheng Xu Link: https://patch.msgid.link/20260917120459.13286-2-chengyou@linux.alibaba.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma_verbs.c | 58 ++++++++++++----------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 65b1af1e6623..57a35f8b32cf 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -512,25 +512,14 @@ static void free_kernel_qp(struct erdma_qp *qp) vfree(qp->kern_qp.swr_tbl); vfree(qp->kern_qp.rwr_tbl); - if (qp->kern_qp.sq_buf) - dma_free_coherent(&dev->pdev->dev, - qp->attrs.sq_size << SQEBB_SHIFT, - qp->kern_qp.sq_buf, - qp->kern_qp.sq_buf_dma_addr); - - if (qp->kern_qp.sq_dbrec) - dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, - qp->kern_qp.sq_dbrec_dma); - - if (qp->kern_qp.rq_buf) - dma_free_coherent(&dev->pdev->dev, - qp->attrs.rq_size << RQE_SHIFT, - qp->kern_qp.rq_buf, - qp->kern_qp.rq_buf_dma_addr); - - if (qp->kern_qp.rq_dbrec) - dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, - qp->kern_qp.rq_dbrec_dma); + dma_free_coherent(&dev->pdev->dev, qp->attrs.sq_size << SQEBB_SHIFT, + qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); + dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, + qp->kern_qp.sq_dbrec_dma); + dma_free_coherent(&dev->pdev->dev, qp->attrs.rq_size << RQE_SHIFT, + qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); + dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, + qp->kern_qp.rq_dbrec_dma); } static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, @@ -551,36 +540,51 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64)); + if (!kqp->swr_tbl) + return -ENOMEM; + kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64)); - if (!kqp->swr_tbl || !kqp->rwr_tbl) - goto err_out; + if (!kqp->rwr_tbl) + goto err_free_swr_tbl; size = qp->attrs.sq_size << SQEBB_SHIFT; kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, &kqp->sq_buf_dma_addr, GFP_KERNEL); if (!kqp->sq_buf) - goto err_out; + goto err_free_rwr_tbl; kqp->sq_dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); if (!kqp->sq_dbrec) - goto err_out; + goto err_free_sq_buf; size = qp->attrs.rq_size << RQE_SHIFT; kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, &kqp->rq_buf_dma_addr, GFP_KERNEL); if (!kqp->rq_buf) - goto err_out; + goto err_free_sq_dbrec; kqp->rq_dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); if (!kqp->rq_dbrec) - goto err_out; + goto err_free_rq_buf; return 0; -err_out: - free_kernel_qp(qp); +err_free_rq_buf: + dma_free_coherent(&dev->pdev->dev, size, kqp->rq_buf, + kqp->rq_buf_dma_addr); +err_free_sq_dbrec: + dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma); +err_free_sq_buf: + size = qp->attrs.sq_size << SQEBB_SHIFT; + dma_free_coherent(&dev->pdev->dev, size, kqp->sq_buf, + kqp->sq_buf_dma_addr); +err_free_rwr_tbl: + vfree(kqp->rwr_tbl); +err_free_swr_tbl: + vfree(kqp->swr_tbl); + return -ENOMEM; } From 97ff4e6cd8d87ecad9da8902d38adc97a9c7150b Mon Sep 17 00:00:00 2001 From: Cheng Xu Date: Thu, 17 Sep 2026 20:04:56 +0800 Subject: [PATCH 049/153] RDMA/erdma: Support non-contiguous kernel QP buffers A single coherent allocation for kernel QP queues can fail for large queues when memory is fragmented. Allocate page-sized coherent buffers and describe them with the existing MTT. Keep the userspace QP path unchanged. Signed-off-by: Cheng Xu Link: https://patch.msgid.link/20260917120459.13286-3-chengyou@linux.alibaba.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma_cq.c | 4 +- drivers/infiniband/hw/erdma/erdma_qp.c | 38 ++-- drivers/infiniband/hw/erdma/erdma_verbs.c | 211 +++++++++++++--------- drivers/infiniband/hw/erdma/erdma_verbs.h | 44 ++++- 4 files changed, 188 insertions(+), 109 deletions(-) diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c index 1f456327e63c..4cc0254b0ab5 100644 --- a/drivers/infiniband/hw/erdma/erdma_cq.c +++ b/drivers/infiniband/hw/erdma/erdma_cq.c @@ -161,8 +161,8 @@ static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc) if (qtype == ERDMA_CQE_QTYPE_SQ) { id_table = kern_qp->swr_tbl; depth = qp->attrs.sq_size; - wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, - qp->attrs.sq_size, SQEBB_SHIFT); + wqe_hdr = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx, + qp->attrs.sq_size, SQEBB_SHIFT); kern_qp->sq_ci = FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) + wqe_idx + 1; diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c index e002343832f7..fc335b674746 100644 --- a/drivers/infiniband/hw/erdma/erdma_qp.c +++ b/drivers/infiniband/hw/erdma/erdma_qp.c @@ -259,8 +259,8 @@ static void erdma_reset_qp(struct erdma_qp *qp) qp->kern_qp.rq_ci = 0; memset(qp->kern_qp.swr_tbl, 0, qp->attrs.sq_size * sizeof(u64)); memset(qp->kern_qp.rwr_tbl, 0, qp->attrs.rq_size * sizeof(u64)); - memset(qp->kern_qp.sq_buf, 0, qp->attrs.sq_size << SQEBB_SHIFT); - memset(qp->kern_qp.rq_buf, 0, qp->attrs.rq_size << RQE_SHIFT); + erdma_kmem_clear(&qp->kern_qp.sq_mem); + erdma_kmem_clear(&qp->kern_qp.rq_mem); erdma_remove_cqes_of_qp(&qp->scq->ibcq, QP_ID(qp)); if (qp->rcq != qp->scq) erdma_remove_cqes_of_qp(&qp->rcq->ibcq, QP_ID(qp)); @@ -332,8 +332,8 @@ static int fill_inline_data(struct erdma_qp *qp, wqe_idx += (sgl_offset >> SQEBB_SHIFT); sgl_offset &= (SQEBB_SIZE - 1); - data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size, - SQEBB_SHIFT); + data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx, + qp->attrs.sq_size, SQEBB_SHIFT); while (i < send_wr->num_sge) { bytes += send_wr->sg_list[i].length; @@ -356,8 +356,9 @@ static int fill_inline_data(struct erdma_qp *qp, wqe_idx += (sgl_offset >> SQEBB_SHIFT); sgl_offset &= (SQEBB_SIZE - 1); - data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, - qp->attrs.sq_size, SQEBB_SHIFT); + data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, + wqe_idx, qp->attrs.sq_size, + SQEBB_SHIFT); if (!remain_size) break; } @@ -385,8 +386,8 @@ static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr, while (i < send_wr->num_sge) { wqe_idx += (sgl_offset >> SQEBB_SHIFT); sgl_offset &= (SQEBB_SIZE - 1); - sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, - qp->attrs.sq_size, SQEBB_SHIFT); + sgl = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx, + qp->attrs.sq_size, SQEBB_SHIFT); bytes += send_wr->sg_list[i].length; memcpy(sgl + sgl_offset, &send_wr->sg_list[i], @@ -463,8 +464,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, send_wr->opcode != IB_WR_SEND_WITH_IMM) return -EINVAL; - entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size, - SQEBB_SHIFT); + entry = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx, + qp->attrs.sq_size, SQEBB_SHIFT); /* Clear the SQE header section. */ *entry = 0; @@ -524,8 +525,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, read_sqe->sink_to_h = cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr)); - sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1, - qp->attrs.sq_size, SQEBB_SHIFT); + sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1, + qp->attrs.sq_size, SQEBB_SHIFT); sge->addr = cpu_to_le64(rdma_wr->remote_addr); sge->key = cpu_to_le32(rdma_wr->rkey); sge->length = cpu_to_le32(send_wr->sg_list[0].length); @@ -569,8 +570,9 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) { attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0); /* Copy SGLs to SQE content to accelerate */ - memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1, - qp->attrs.sq_size, SQEBB_SHIFT), + memcpy(erdma_kmem_get_entry(&qp->kern_qp.sq_mem, + idx + 1, qp->attrs.sq_size, + SQEBB_SHIFT), mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents)); wqe_size = sizeof(struct erdma_reg_mr_sqe) + MTT_SIZE(mr->mem.mtt_nents); @@ -605,8 +607,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, cpu_to_le64(atomic_wr(send_wr)->compare_add); } - sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1, - qp->attrs.sq_size, SQEBB_SHIFT); + sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1, + qp->attrs.sq_size, SQEBB_SHIFT); sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr); sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey); sge++; @@ -702,8 +704,8 @@ static int erdma_post_recv_one(struct erdma_qp *qp, const struct ib_recv_wr *recv_wr) { struct erdma_rqe *rqe = - get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi, - qp->attrs.rq_size, RQE_SHIFT); + erdma_kmem_get_entry(&qp->kern_qp.rq_mem, qp->kern_qp.rq_pi, + qp->attrs.rq_size, RQE_SHIFT); rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1); rqe->qpn = cpu_to_le32(QP_ID(qp)); diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 57a35f8b32cf..ec29701675ca 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -19,6 +19,10 @@ #include "erdma_cm.h" #include "erdma_verbs.h" +static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem, + size_t size); +static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem); + static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg, u64 *addr0, u64 *addr1) { @@ -41,7 +45,7 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) struct erdma_dev *dev = to_edev(qp->ibqp.device); struct erdma_pd *pd = to_epd(qp->ibqp.pd); struct erdma_cmdq_create_qp_req req; - struct erdma_uqp *user_qp; + struct erdma_mem *sq_mem, *rq_mem; u64 resp0, resp1; int err; @@ -63,67 +67,46 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) ERDMA_QPT_UD); if (rdma_is_kernel_res(&qp->ibqp.res)) { - u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT; - - req.sq_cqn_mtt_cfg = - FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, - pgsz_range) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); - req.rq_cqn_mtt_cfg = - FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, - pgsz_range) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); - - req.sq_mtt_cfg = - FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK, - ERDMA_MR_MTT_0LEVEL); - req.rq_mtt_cfg = req.sq_mtt_cfg; - - req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; - req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; + sq_mem = &qp->kern_qp.sq_mem; + rq_mem = &qp->kern_qp.rq_mem; req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma; req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma; } else { - user_qp = &qp->user_qp; - req.sq_cqn_mtt_cfg = FIELD_PREP( - ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, - ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT); + sq_mem = &qp->user_qp.sq_mem; + rq_mem = &qp->user_qp.rq_mem; + req.sq_dbrec_dma = qp->user_qp.sq_dbrec_dma; + req.rq_dbrec_dma = qp->user_qp.rq_dbrec_dma; + } + + req.sq_cqn_mtt_cfg = + FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, + ilog2(sq_mem->page_size) - ERDMA_HW_PAGE_SHIFT); + req.sq_cqn_mtt_cfg |= + FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); + req.rq_cqn_mtt_cfg = + FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, + ilog2(rq_mem->page_size) - ERDMA_HW_PAGE_SHIFT); + req.rq_cqn_mtt_cfg |= + FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); + + req.sq_mtt_cfg = + sq_mem->page_offset | + FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, sq_mem->mtt_nents); + req.rq_mtt_cfg = + rq_mem->page_offset | + FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, rq_mem->mtt_nents); + + assemble_qbuf_mtt_for_cmd(sq_mem, &req.sq_mtt_cfg, &req.sq_buf_addr, + req.sq_mtt_entry); + assemble_qbuf_mtt_for_cmd(rq_mem, &req.rq_mtt_cfg, &req.rq_buf_addr, + req.rq_mtt_entry); + if (uctx && uctx->ext_db.enable) { req.sq_cqn_mtt_cfg |= - FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); - - req.rq_cqn_mtt_cfg = FIELD_PREP( - ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, - ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT); - req.rq_cqn_mtt_cfg |= - FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); - - req.sq_mtt_cfg = user_qp->sq_mem.page_offset; - req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, - user_qp->sq_mem.mtt_nents); - - req.rq_mtt_cfg = user_qp->rq_mem.page_offset; - req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, - user_qp->rq_mem.mtt_nents); - - assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg, - &req.sq_buf_addr, req.sq_mtt_entry); - assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg, - &req.rq_buf_addr, req.rq_mtt_entry); - - req.sq_dbrec_dma = user_qp->sq_dbrec_dma; - req.rq_dbrec_dma = user_qp->rq_dbrec_dma; - - if (uctx->ext_db.enable) { - req.sq_cqn_mtt_cfg |= - FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1); - req.db_cfg = - FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK, - uctx->ext_db.sdb_off) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK, - uctx->ext_db.rdb_off); - } + FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1); + req.db_cfg = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK, + uctx->ext_db.sdb_off) | + FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK, + uctx->ext_db.rdb_off); } err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1, @@ -512,12 +495,10 @@ static void free_kernel_qp(struct erdma_qp *qp) vfree(qp->kern_qp.swr_tbl); vfree(qp->kern_qp.rwr_tbl); - dma_free_coherent(&dev->pdev->dev, qp->attrs.sq_size << SQEBB_SHIFT, - qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); + put_mtt_entries(dev, &qp->kern_qp.sq_mem); dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, qp->kern_qp.sq_dbrec_dma); - dma_free_coherent(&dev->pdev->dev, qp->attrs.rq_size << RQE_SHIFT, - qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); + put_mtt_entries(dev, &qp->kern_qp.rq_mem); dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, qp->kern_qp.rq_dbrec_dma); } @@ -526,6 +507,7 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, struct ib_qp_init_attr *attrs) { struct erdma_kqp *kqp = &qp->kern_qp; + int ret = -ENOMEM; int size; if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) @@ -548,44 +530,43 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, goto err_free_swr_tbl; size = qp->attrs.sq_size << SQEBB_SHIFT; - kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, - &kqp->sq_buf_dma_addr, GFP_KERNEL); - if (!kqp->sq_buf) + ret = erdma_alloc_kmem(dev, &kqp->sq_mem, size); + if (ret) goto err_free_rwr_tbl; kqp->sq_dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); - if (!kqp->sq_dbrec) - goto err_free_sq_buf; + if (!kqp->sq_dbrec) { + ret = -ENOMEM; + goto err_free_sq_mem; + } size = qp->attrs.rq_size << RQE_SHIFT; - kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, - &kqp->rq_buf_dma_addr, GFP_KERNEL); - if (!kqp->rq_buf) + ret = erdma_alloc_kmem(dev, &kqp->rq_mem, size); + if (ret) goto err_free_sq_dbrec; kqp->rq_dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); - if (!kqp->rq_dbrec) - goto err_free_rq_buf; + if (!kqp->rq_dbrec) { + ret = -ENOMEM; + goto err_free_rq_mem; + } return 0; -err_free_rq_buf: - dma_free_coherent(&dev->pdev->dev, size, kqp->rq_buf, - kqp->rq_buf_dma_addr); +err_free_rq_mem: + put_mtt_entries(dev, &kqp->rq_mem); err_free_sq_dbrec: dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma); -err_free_sq_buf: - size = qp->attrs.sq_size << SQEBB_SHIFT; - dma_free_coherent(&dev->pdev->dev, size, kqp->sq_buf, - kqp->sq_buf_dma_addr); +err_free_sq_mem: + put_mtt_entries(dev, &kqp->sq_mem); err_free_rwr_tbl: vfree(kqp->rwr_tbl); err_free_swr_tbl: vfree(kqp->swr_tbl); - return -ENOMEM; + return ret; } static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem) @@ -830,12 +811,76 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt) } } +static void erdma_free_kmem(struct erdma_dev *dev, struct erdma_mem *mem) +{ + struct erdma_buf_list *buf_list = mem->kmem.buf_list; + u32 i; + + for (i = 0; i < mem->page_cnt; i++) + dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf, + buf_list[i].dma_addr); + + kfree(buf_list); +} + +static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem, + size_t size) +{ + struct erdma_buf_list *buf_list; + struct erdma_mtt *mtt; + int ret = -ENOMEM; + u32 i; + + mem->type = ERDMA_KMEM; + mem->page_size = PAGE_SIZE; + mem->page_offset = 0; + mem->page_cnt = DIV_ROUND_UP(size, PAGE_SIZE); + mem->mtt_nents = mem->page_cnt; + mem->len = size; + + buf_list = kcalloc(mem->page_cnt, sizeof(*buf_list), GFP_KERNEL); + if (!buf_list) + return -ENOMEM; + + for (i = 0; i < mem->page_cnt; i++) { + buf_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, + &buf_list[i].dma_addr, + GFP_KERNEL); + if (!buf_list[i].buf) + goto err_free_pages; + } + + mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), true); + if (IS_ERR(mtt)) { + ret = PTR_ERR(mtt); + goto err_free_pages; + } + + for (i = 0; i < mem->page_cnt; i++) + mtt->buf[i] = buf_list[i].dma_addr; + dma_sync_single_for_device(&dev->pdev->dev, mtt->buf_dma, mtt->size, + DMA_TO_DEVICE); + mem->kmem.buf_list = buf_list; + mem->mtt = mtt; + + return 0; + +err_free_pages: + while (i--) + dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf, + buf_list[i].dma_addr); + kfree(buf_list); + + return ret; +} + static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, u64 start, u64 len, int access, u64 virt, unsigned long req_page_size, bool force_continuous) { int ret = 0; + mem->type = ERDMA_UMEM; mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access); if (IS_ERR(mem->umem)) { ret = PTR_ERR(mem->umem); @@ -874,10 +919,10 @@ static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem) if (mem->mtt) erdma_destroy_mtt(dev, mem->mtt); - if (mem->umem) { + if (mem->type == ERDMA_KMEM) + erdma_free_kmem(dev, mem); + else if (mem->umem) ib_umem_release(mem->umem); - mem->umem = NULL; - } } static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index c73cecf92f61..401e76e846b9 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -111,8 +111,26 @@ struct erdma_mtt { struct erdma_mtt *low_level; }; +enum erdma_mem_type { + ERDMA_UMEM = 0, + ERDMA_KMEM = 1, +}; + +struct erdma_buf_list { + void *buf; + dma_addr_t dma_addr; +}; + +struct erdma_kmem { + struct erdma_buf_list *buf_list; +}; + struct erdma_mem { - struct ib_umem *umem; + enum erdma_mem_type type; + union { + struct ib_umem *umem; + struct erdma_kmem kmem; + }; struct erdma_mtt *mtt; u32 page_size; @@ -124,6 +142,23 @@ struct erdma_mem { u64 len; }; +static inline void *erdma_kmem_get_entry(struct erdma_mem *mem, u32 idx, + u32 depth, u32 shift) +{ + u32 offset = (idx & (depth - 1)) << shift; + + return (u8 *)mem->kmem.buf_list[offset >> PAGE_SHIFT].buf + + offset_in_page(offset); +} + +static inline void erdma_kmem_clear(struct erdma_mem *mem) +{ + u32 i; + + for (i = 0; i < mem->page_cnt; i++) + memset(mem->kmem.buf_list[i].buf, 0, PAGE_SIZE); +} + struct erdma_mr { struct ib_mr ibmr; struct erdma_mem mem; @@ -183,11 +218,8 @@ struct erdma_kqp { void __iomem *hw_sq_db; void __iomem *hw_rq_db; - void *sq_buf; - dma_addr_t sq_buf_dma_addr; - - void *rq_buf; - dma_addr_t rq_buf_dma_addr; + struct erdma_mem sq_mem; + struct erdma_mem rq_mem; void *sq_dbrec; void *rq_dbrec; From 717ba25e0904bf3cdf331dc0d8ad71c88b71f696 Mon Sep 17 00:00:00 2001 From: Cheng Xu Date: Thu, 17 Sep 2026 20:04:57 +0800 Subject: [PATCH 050/153] RDMA/erdma: Support non-contiguous kernel CQ buffers A single coherent allocation for a kernel CQ can fail when memory is fragmented. Use page-sized coherent buffers and describe them with the existing MTT. Keep the userspace CQ path and doorbell allocation unchanged. Signed-off-by: Cheng Xu Link: https://patch.msgid.link/20260917120459.13286-4-chengyou@linux.alibaba.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma_cq.c | 14 ++--- drivers/infiniband/hw/erdma/erdma_verbs.c | 76 +++++++++-------------- drivers/infiniband/hw/erdma/erdma_verbs.h | 3 +- 3 files changed, 36 insertions(+), 57 deletions(-) diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c index 4cc0254b0ab5..daec74f0aa6d 100644 --- a/drivers/infiniband/hw/erdma/erdma_cq.c +++ b/drivers/infiniband/hw/erdma/erdma_cq.c @@ -8,8 +8,8 @@ static void *get_next_valid_cqe(struct erdma_cq *cq) { - __be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci, - cq->depth, CQE_SHIFT); + __be32 *cqe = erdma_kmem_get_entry( + &cq->kern_cq.qbuf_mem, cq->kern_cq.ci, cq->depth, CQE_SHIFT); u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK, be32_to_cpu(READ_ONCE(*cqe))); @@ -242,15 +242,15 @@ void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn) while (ncqe > 0) { cur_cq_ci = prev_cq_ci + ncqe - 1; - cqe = get_queue_entry(cq->kern_cq.qbuf, cur_cq_ci, cq->depth, - CQE_SHIFT); + cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem, cur_cq_ci, + cq->depth, CQE_SHIFT); if (be32_to_cpu(cqe->qpn) == qpn) { ++nqp_cqe; } else if (nqp_cqe) { - dst_cqe = get_queue_entry(cq->kern_cq.qbuf, - cur_cq_ci + nqp_cqe, - cq->depth, CQE_SHIFT); + dst_cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem, + cur_cq_ci + nqp_cqe, + cq->depth, CQE_SHIFT); owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK, be32_to_cpu(dst_cqe->hdr)); cqe->hdr = cpu_to_be32( diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index ec29701675ca..345b75377681 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -177,7 +177,6 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) struct erdma_dev *dev = to_edev(cq->ibcq.device); struct erdma_cmdq_create_cq_req req; struct erdma_mem *mem; - u32 page_size; erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_CREATE_CQ); @@ -187,48 +186,34 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn); if (rdma_is_kernel_res(&cq->ibcq.res)) { - page_size = SZ_32M; - req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, - ilog2(page_size) - ERDMA_HW_PAGE_SHIFT); - req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr); - req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr); - - req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) | - FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, - ERDMA_MR_MTT_0LEVEL); - - req.first_page_offset = 0; + mem = &cq->kern_cq.qbuf_mem; req.cq_dbrec_dma = cq->kern_cq.dbrec_dma; } else { mem = &cq->user_cq.qbuf_mem; - req.cfg0 |= - FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, - ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT); - if (mem->mtt_nents == 1) { - req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]); - req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]); - req.cfg1 |= - FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, - ERDMA_MR_MTT_0LEVEL); - } else { - req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma); - req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma); - req.cfg1 |= - FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, - ERDMA_MR_MTT_1LEVEL); - } - req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, - mem->mtt_nents); - - req.first_page_offset = mem->page_offset; req.cq_dbrec_dma = cq->user_cq.dbrec_dma; + } - if (uctx->ext_db.enable) { - req.cfg1 |= FIELD_PREP( - ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1); - req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK, - uctx->ext_db.cdb_off); - } + req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, + ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT); + if (mem->mtt_nents == 1) { + req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]); + req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]); + req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, + ERDMA_MR_MTT_0LEVEL); + } else { + req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma); + req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma); + req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, + ERDMA_MR_MTT_1LEVEL); + } + req.cfg1 |= + FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, mem->mtt_nents); + req.first_page_offset = mem->page_offset; + + if (uctx && uctx->ext_db.enable) { + req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1); + req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK, + uctx->ext_db.cdb_off); } return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, @@ -1398,8 +1383,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) wait_for_completion(&cq->free); if (rdma_is_kernel_res(&cq->ibcq.res)) { - dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, - cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + put_mtt_entries(dev, &cq->kern_cq.qbuf_mem); dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, cq->kern_cq.dbrec_dma); } else { @@ -1997,10 +1981,8 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq) { struct erdma_dev *dev = to_edev(cq->ibcq.device); - cq->kern_cq.qbuf = - dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, - &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); - if (!cq->kern_cq.qbuf) + if (erdma_alloc_kmem(dev, &cq->kern_cq.qbuf_mem, + cq->depth << CQE_SHIFT)) return -ENOMEM; cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, @@ -2015,8 +1997,7 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq) return 0; err_out: - dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, - cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + put_mtt_entries(dev, &cq->kern_cq.qbuf_mem); return -ENOMEM; } @@ -2083,8 +2064,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); put_mtt_entries(dev, &cq->user_cq.qbuf_mem); } else { - dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT, - cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + put_mtt_entries(dev, &cq->kern_cq.qbuf_mem); dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, cq->kern_cq.dbrec_dma); } diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index 401e76e846b9..28e898f1b6f5 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -352,8 +352,7 @@ struct erdma_qp { }; struct erdma_kcq_info { - void *qbuf; - dma_addr_t qbuf_dma_addr; + struct erdma_mem qbuf_mem; u32 ci; u32 cmdsn; u32 notify_cnt; From 30cc22441650413ad8412e3e2b11961459c331ba Mon Sep 17 00:00:00 2001 From: Cheng Xu Date: Thu, 17 Sep 2026 20:04:58 +0800 Subject: [PATCH 051/153] RDMA/erdma: Unify userspace and kernel queue buffer management Userspace and kernel space queue buffers use separate helpers despite sharing MTT metadata and lifetime rules. Manage both through erdma_mem_init() and erdma_mem_uninit(), while keeping backing allocation and release type-specific. Signed-off-by: Cheng Xu Link: https://patch.msgid.link/20260917120459.13286-5-chengyou@linux.alibaba.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma_verbs.c | 268 +++++++++++++--------- drivers/infiniband/hw/erdma/erdma_verbs.h | 22 +- 2 files changed, 175 insertions(+), 115 deletions(-) diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 345b75377681..1ff31c4587bd 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -19,9 +19,9 @@ #include "erdma_cm.h" #include "erdma_verbs.h" -static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem, - size_t size); -static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem); +static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem, + struct erdma_mem_init_attr *attr); +static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem); static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg, u64 *addr0, u64 *addr1) @@ -480,10 +480,10 @@ static void free_kernel_qp(struct erdma_qp *qp) vfree(qp->kern_qp.swr_tbl); vfree(qp->kern_qp.rwr_tbl); - put_mtt_entries(dev, &qp->kern_qp.sq_mem); + erdma_mem_uninit(dev, &qp->kern_qp.sq_mem); dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, qp->kern_qp.sq_dbrec_dma); - put_mtt_entries(dev, &qp->kern_qp.rq_mem); + erdma_mem_uninit(dev, &qp->kern_qp.rq_mem); dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, qp->kern_qp.rq_dbrec_dma); } @@ -491,6 +491,9 @@ static void free_kernel_qp(struct erdma_qp *qp) static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, struct ib_qp_init_attr *attrs) { + struct erdma_mem_init_attr attr = { + .type = ERDMA_KMEM, + }; struct erdma_kqp *kqp = &qp->kern_qp; int ret = -ENOMEM; int size; @@ -515,7 +518,8 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, goto err_free_swr_tbl; size = qp->attrs.sq_size << SQEBB_SHIFT; - ret = erdma_alloc_kmem(dev, &kqp->sq_mem, size); + attr.len = size; + ret = erdma_mem_init(dev, &kqp->sq_mem, &attr); if (ret) goto err_free_rwr_tbl; @@ -527,7 +531,8 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, } size = qp->attrs.rq_size << RQE_SHIFT; - ret = erdma_alloc_kmem(dev, &kqp->rq_mem, size); + attr.len = size; + ret = erdma_mem_init(dev, &kqp->rq_mem, &attr); if (ret) goto err_free_sq_dbrec; @@ -541,11 +546,11 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, return 0; err_free_rq_mem: - put_mtt_entries(dev, &kqp->rq_mem); + erdma_mem_uninit(dev, &kqp->rq_mem); err_free_sq_dbrec: dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma); err_free_sq_mem: - put_mtt_entries(dev, &kqp->sq_mem); + erdma_mem_uninit(dev, &kqp->sq_mem); err_free_rwr_tbl: vfree(kqp->rwr_tbl); err_free_swr_tbl: @@ -554,17 +559,28 @@ static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, return ret; } -static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem) +static void erdma_init_mtt_leaf(struct erdma_mem *mem, struct erdma_mtt *mtt) { - struct erdma_mtt *mtt = mem->mtt; struct ib_block_iter biter; u32 idx = 0; + if (mem->type == ERDMA_UMEM) { + rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) + mtt->buf[idx++] = rdma_block_iter_dma_address(&biter); + } else { + for (; idx < mem->page_cnt; idx++) + mtt->buf[idx] = mem->kmem.buf_list[idx].dma_addr; + } +} + +static void erdma_init_bottom_mtt(struct erdma_mem *mem) +{ + struct erdma_mtt *mtt = mem->mtt; + while (mtt->low_level) mtt = mtt->low_level; - rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) - mtt->buf[idx++] = rdma_block_iter_dma_address(&biter); + erdma_init_mtt_leaf(mem, mtt); } static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev, @@ -796,7 +812,7 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt) } } -static void erdma_free_kmem(struct erdma_dev *dev, struct erdma_mem *mem) +static void erdma_kmem_free(struct erdma_dev *dev, struct erdma_mem *mem) { struct erdma_buf_list *buf_list = mem->kmem.buf_list; u32 i; @@ -808,26 +824,17 @@ static void erdma_free_kmem(struct erdma_dev *dev, struct erdma_mem *mem) kfree(buf_list); } -static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem, - size_t size) +static int erdma_kmem_alloc(struct erdma_dev *dev, struct erdma_mem *mem, + u32 page_cnt) { struct erdma_buf_list *buf_list; - struct erdma_mtt *mtt; - int ret = -ENOMEM; u32 i; - mem->type = ERDMA_KMEM; - mem->page_size = PAGE_SIZE; - mem->page_offset = 0; - mem->page_cnt = DIV_ROUND_UP(size, PAGE_SIZE); - mem->mtt_nents = mem->page_cnt; - mem->len = size; - - buf_list = kcalloc(mem->page_cnt, sizeof(*buf_list), GFP_KERNEL); + buf_list = kcalloc(page_cnt, sizeof(*buf_list), GFP_KERNEL); if (!buf_list) return -ENOMEM; - for (i = 0; i < mem->page_cnt; i++) { + for (i = 0; i < page_cnt; i++) { buf_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, &buf_list[i].dma_addr, GFP_KERNEL); @@ -835,19 +842,7 @@ static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem, goto err_free_pages; } - mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), true); - if (IS_ERR(mtt)) { - ret = PTR_ERR(mtt); - goto err_free_pages; - } - - for (i = 0; i < mem->page_cnt; i++) - mtt->buf[i] = buf_list[i].dma_addr; - dma_sync_single_for_device(&dev->pdev->dev, mtt->buf_dma, mtt->size, - DMA_TO_DEVICE); mem->kmem.buf_list = buf_list; - mem->mtt = mtt; - return 0; err_free_pages: @@ -856,58 +851,85 @@ static int erdma_alloc_kmem(struct erdma_dev *dev, struct erdma_mem *mem, buf_list[i].dma_addr); kfree(buf_list); - return ret; + return -ENOMEM; } -static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, - u64 start, u64 len, int access, u64 virt, - unsigned long req_page_size, bool force_continuous) +static void erdma_mem_free(struct erdma_dev *dev, struct erdma_mem *mem) { - int ret = 0; + switch (mem->type) { + case ERDMA_UMEM: + ib_umem_release(mem->umem); + break; + case ERDMA_KMEM: + erdma_kmem_free(dev, mem); + break; + default: + break; + } +} - mem->type = ERDMA_UMEM; - mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access); - if (IS_ERR(mem->umem)) { - ret = PTR_ERR(mem->umem); - mem->umem = NULL; - return ret; +static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem, + struct erdma_mem_init_attr *attr) +{ + struct erdma_mtt *mtt; + int ret; + + mem->type = attr->type; + + switch (mem->type) { + case ERDMA_UMEM: + mem->umem = ib_umem_get_va(&dev->ibdev, attr->start, attr->len, + attr->access); + if (IS_ERR(mem->umem)) { + ret = PTR_ERR(mem->umem); + return ret; + } + + mem->page_size = ib_umem_find_best_pgsz( + mem->umem, attr->req_page_size, attr->virt); + mem->mtt_nents = + ib_umem_num_dma_blocks(mem->umem, mem->page_size); + break; + case ERDMA_KMEM: + mem->page_size = PAGE_SIZE; + mem->mtt_nents = DIV_ROUND_UP(attr->len, PAGE_SIZE); + ret = erdma_kmem_alloc(dev, mem, mem->mtt_nents); + if (ret) + return ret; + break; + default: + return -EINVAL; } - mem->va = virt; - mem->len = len; - mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt); - mem->page_offset = start & (mem->page_size - 1); - mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size); + mem->va = attr->virt; + mem->len = attr->len; + mem->page_offset = attr->start & (mem->page_size - 1); mem->page_cnt = mem->mtt_nents; - mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), - force_continuous); - if (IS_ERR(mem->mtt)) { - ret = PTR_ERR(mem->mtt); - goto error_ret; + mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), + !(attr->flags & ERDMA_MEM_FLAG_MR_BUF)); + if (IS_ERR(mtt)) { + ret = PTR_ERR(mtt); + goto err_free_mem; } + mem->mtt = mtt; - erdma_fill_bottom_mtt(dev, mem); + erdma_init_bottom_mtt(mem); + if (mtt->continuous) + dma_sync_single_for_device(&dev->pdev->dev, mtt->buf_dma, + mtt->size, DMA_TO_DEVICE); return 0; -error_ret: - if (mem->umem) { - ib_umem_release(mem->umem); - mem->umem = NULL; - } +err_free_mem: + erdma_mem_free(dev, mem); return ret; } -static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem) +static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem) { - if (mem->mtt) - erdma_destroy_mtt(dev, mem->mtt); - - if (mem->type == ERDMA_KMEM) - erdma_free_kmem(dev, mem); - else if (mem->umem) - ib_umem_release(mem->umem); + erdma_destroy_mtt(dev, mem->mtt); + erdma_mem_free(dev, mem); } static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, @@ -975,6 +997,9 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, u64 va, u32 len, u64 dbrec_va) { + struct erdma_mem_init_attr attr = { + .type = ERDMA_UMEM, + }; dma_addr_t dbrec_dma; u32 rq_offset; int ret; @@ -983,45 +1008,47 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, qp->attrs.rq_size * RQE_SIZE)) return -EINVAL; - ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va, - qp->attrs.sq_size << SQEBB_SHIFT, 0, va, - (SZ_1M - SZ_4K), true); + attr.start = va; + attr.virt = va; + attr.len = qp->attrs.sq_size << SQEBB_SHIFT; + attr.req_page_size = SZ_1M - SZ_4K; + ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, &attr); if (ret) return ret; rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE); qp->user_qp.rq_offset = rq_offset; - ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset, - qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset, - (SZ_1M - SZ_4K), true); + attr.start = va + rq_offset; + attr.virt = va + rq_offset; + attr.len = qp->attrs.rq_size << RQE_SHIFT; + ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, &attr); if (ret) - goto put_sq_mtt; + goto uninit_sq_mem; ret = erdma_map_user_dbrecords(uctx, dbrec_va, - &qp->user_qp.user_dbr_page, - &dbrec_dma); + &qp->user_qp.user_dbr_page, &dbrec_dma); if (ret) - goto put_rq_mtt; + goto uninit_rq_mem; qp->user_qp.sq_dbrec_dma = dbrec_dma; qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE; return 0; -put_rq_mtt: - put_mtt_entries(qp->dev, &qp->user_qp.rq_mem); +uninit_rq_mem: + erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem); -put_sq_mtt: - put_mtt_entries(qp->dev, &qp->user_qp.sq_mem); +uninit_sq_mem: + erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem); return ret; } static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx) { - put_mtt_entries(qp->dev, &qp->user_qp.sq_mem); - put_mtt_entries(qp->dev, &qp->user_qp.rq_mem); + erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem); + erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem); erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page); } @@ -1268,8 +1295,11 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, u64 virt, int access, struct ib_dmah *dmah, struct ib_udata *udata) { - struct erdma_mr *mr = NULL; struct erdma_dev *dev = to_edev(ibpd->device); + struct erdma_mem_init_attr attr = { + .type = ERDMA_UMEM, + }; + struct erdma_mr *mr = NULL; u32 stag; int ret; @@ -1283,8 +1313,13 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, if (!mr) return ERR_PTR(-ENOMEM); - ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt, - SZ_2G - SZ_4K, false); + attr.start = start; + attr.virt = virt; + attr.len = len; + attr.req_page_size = SZ_2G - SZ_4K; + attr.access = access; + attr.flags = ERDMA_MEM_FLAG_MR_BUF; + ret = erdma_mem_init(dev, &mr->mem, &attr); if (ret) goto err_out_free; @@ -1294,8 +1329,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr->ibmr.lkey = mr->ibmr.rkey = stag; mr->ibmr.pd = ibpd; - mr->mem.va = virt; - mr->mem.len = len; mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access); mr->valid = 1; mr->type = ERDMA_MR_TYPE_NORMAL; @@ -1311,7 +1344,7 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr->ibmr.lkey >> 8); err_out_put_mtt: - put_mtt_entries(dev, &mr->mem); + erdma_mem_uninit(dev, &mr->mem); err_out_free: kfree(mr); @@ -1348,7 +1381,8 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); - put_mtt_entries(dev, &mr->mem); + if (mr->type != ERDMA_MR_TYPE_DMA) + erdma_mem_uninit(dev, &mr->mem); kfree(mr); return 0; @@ -1383,12 +1417,12 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) wait_for_completion(&cq->free); if (rdma_is_kernel_res(&cq->ibcq.res)) { - put_mtt_entries(dev, &cq->kern_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem); dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, cq->kern_cq.dbrec_dma); } else { erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); - put_mtt_entries(dev, &cq->user_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem); } return 0; @@ -1440,8 +1474,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (rdma_is_kernel_res(&qp->ibqp.res)) { free_kernel_qp(qp); } else { - put_mtt_entries(dev, &qp->user_qp.sq_mem); - put_mtt_entries(dev, &qp->user_qp.rq_mem); + erdma_mem_uninit(dev, &qp->user_qp.sq_mem); + erdma_mem_uninit(dev, &qp->user_qp.rq_mem); erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page); } @@ -1959,12 +1993,17 @@ int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, struct erdma_ureq_create_cq *ureq) { - int ret; struct erdma_dev *dev = to_edev(cq->ibcq.device); + struct erdma_mem_init_attr attr = { + .type = ERDMA_UMEM, + }; + int ret; - ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va, - ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K, - true); + attr.start = ureq->qbuf_va; + attr.virt = ureq->qbuf_va; + attr.len = ureq->qbuf_len; + attr.req_page_size = SZ_64M - SZ_4K; + ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr); if (ret) return ret; @@ -1972,7 +2011,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, &cq->user_cq.user_dbr_page, &cq->user_cq.dbrec_dma); if (ret) - put_mtt_entries(dev, &cq->user_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem); return ret; } @@ -1980,10 +2019,15 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, static int erdma_init_kernel_cq(struct erdma_cq *cq) { struct erdma_dev *dev = to_edev(cq->ibcq.device); + struct erdma_mem_init_attr attr = { + .type = ERDMA_KMEM, + .len = cq->depth << CQE_SHIFT, + }; + int ret; - if (erdma_alloc_kmem(dev, &cq->kern_cq.qbuf_mem, - cq->depth << CQE_SHIFT)) - return -ENOMEM; + ret = erdma_mem_init(dev, &cq->kern_cq.qbuf_mem, &attr); + if (ret) + return ret; cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &cq->kern_cq.dbrec_dma); @@ -1997,7 +2041,7 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq) return 0; err_out: - put_mtt_entries(dev, &cq->kern_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem); return -ENOMEM; } @@ -2062,9 +2106,9 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, err_free_res: if (!rdma_is_kernel_res(&ibcq->res)) { erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); - put_mtt_entries(dev, &cq->user_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem); } else { - put_mtt_entries(dev, &cq->kern_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem); dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, cq->kern_cq.dbrec_dma); } diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index 28e898f1b6f5..02bf3723680d 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -112,8 +112,9 @@ struct erdma_mtt { }; enum erdma_mem_type { - ERDMA_UMEM = 0, - ERDMA_KMEM = 1, + ERDMA_NO_MEM = 0, + ERDMA_UMEM = 1, + ERDMA_KMEM = 2, }; struct erdma_buf_list { @@ -125,19 +126,34 @@ struct erdma_kmem { struct erdma_buf_list *buf_list; }; +enum erdma_mem_flags { + ERDMA_MEM_FLAG_MR_BUF = (1 << 0), +}; + +struct erdma_mem_init_attr { + enum erdma_mem_type type; + u64 start; + u64 virt; + u64 len; + unsigned long req_page_size; + int access; + u32 flags; +}; + struct erdma_mem { enum erdma_mem_type type; union { struct ib_umem *umem; struct erdma_kmem kmem; }; - struct erdma_mtt *mtt; u32 page_size; u32 page_offset; u32 page_cnt; u32 mtt_nents; + struct erdma_mtt *mtt; + u64 va; u64 len; }; From 337b40e287b7e93d1bb350180b529ae927ab7099 Mon Sep 17 00:00:00 2001 From: Cheng Xu Date: Thu, 17 Sep 2026 20:04:59 +0800 Subject: [PATCH 052/153] RDMA/erdma: Move kernel QP helpers after memory helpers Move the kernel QP helpers after the shared memory helpers to remove forward declarations. Signed-off-by: Cheng Xu Link: https://patch.msgid.link/20260917120459.13286-6-chengyou@linux.alibaba.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma_verbs.c | 176 +++++++++++----------- 1 file changed, 86 insertions(+), 90 deletions(-) diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 1ff31c4587bd..80d61430d2a4 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -19,10 +19,6 @@ #include "erdma_cm.h" #include "erdma_verbs.h" -static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem, - struct erdma_mem_init_attr *attr); -static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem); - static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg, u64 *addr0, u64 *addr1) { @@ -473,92 +469,6 @@ static int erdma_qp_validate_attr(struct erdma_dev *dev, return 0; } -static void free_kernel_qp(struct erdma_qp *qp) -{ - struct erdma_dev *dev = qp->dev; - - vfree(qp->kern_qp.swr_tbl); - vfree(qp->kern_qp.rwr_tbl); - - erdma_mem_uninit(dev, &qp->kern_qp.sq_mem); - dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, - qp->kern_qp.sq_dbrec_dma); - erdma_mem_uninit(dev, &qp->kern_qp.rq_mem); - dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, - qp->kern_qp.rq_dbrec_dma); -} - -static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, - struct ib_qp_init_attr *attrs) -{ - struct erdma_mem_init_attr attr = { - .type = ERDMA_KMEM, - }; - struct erdma_kqp *kqp = &qp->kern_qp; - int ret = -ENOMEM; - int size; - - if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) - kqp->sig_all = 1; - - kqp->sq_pi = 0; - kqp->sq_ci = 0; - kqp->rq_pi = 0; - kqp->rq_ci = 0; - kqp->hw_sq_db = - dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); - kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; - - kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64)); - if (!kqp->swr_tbl) - return -ENOMEM; - - kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64)); - if (!kqp->rwr_tbl) - goto err_free_swr_tbl; - - size = qp->attrs.sq_size << SQEBB_SHIFT; - attr.len = size; - ret = erdma_mem_init(dev, &kqp->sq_mem, &attr); - if (ret) - goto err_free_rwr_tbl; - - kqp->sq_dbrec = - dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); - if (!kqp->sq_dbrec) { - ret = -ENOMEM; - goto err_free_sq_mem; - } - - size = qp->attrs.rq_size << RQE_SHIFT; - attr.len = size; - ret = erdma_mem_init(dev, &kqp->rq_mem, &attr); - if (ret) - goto err_free_sq_dbrec; - - kqp->rq_dbrec = - dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); - if (!kqp->rq_dbrec) { - ret = -ENOMEM; - goto err_free_rq_mem; - } - - return 0; - -err_free_rq_mem: - erdma_mem_uninit(dev, &kqp->rq_mem); -err_free_sq_dbrec: - dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma); -err_free_sq_mem: - erdma_mem_uninit(dev, &kqp->sq_mem); -err_free_rwr_tbl: - vfree(kqp->rwr_tbl); -err_free_swr_tbl: - vfree(kqp->swr_tbl); - - return ret; -} - static void erdma_init_mtt_leaf(struct erdma_mem *mem, struct erdma_mtt *mtt) { struct ib_block_iter biter; @@ -932,6 +842,92 @@ static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem) erdma_mem_free(dev, mem); } +static void free_kernel_qp(struct erdma_qp *qp) +{ + struct erdma_dev *dev = qp->dev; + + vfree(qp->kern_qp.swr_tbl); + vfree(qp->kern_qp.rwr_tbl); + + erdma_mem_uninit(dev, &qp->kern_qp.sq_mem); + dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, + qp->kern_qp.sq_dbrec_dma); + erdma_mem_uninit(dev, &qp->kern_qp.rq_mem); + dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, + qp->kern_qp.rq_dbrec_dma); +} + +static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, + struct ib_qp_init_attr *attrs) +{ + struct erdma_mem_init_attr attr = { + .type = ERDMA_KMEM, + }; + struct erdma_kqp *kqp = &qp->kern_qp; + int ret = -ENOMEM; + int size; + + if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) + kqp->sig_all = 1; + + kqp->sq_pi = 0; + kqp->sq_ci = 0; + kqp->rq_pi = 0; + kqp->rq_ci = 0; + kqp->hw_sq_db = + dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); + kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; + + kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64)); + if (!kqp->swr_tbl) + return -ENOMEM; + + kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64)); + if (!kqp->rwr_tbl) + goto err_free_swr_tbl; + + size = qp->attrs.sq_size << SQEBB_SHIFT; + attr.len = size; + ret = erdma_mem_init(dev, &kqp->sq_mem, &attr); + if (ret) + goto err_free_rwr_tbl; + + kqp->sq_dbrec = + dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); + if (!kqp->sq_dbrec) { + ret = -ENOMEM; + goto err_free_sq_mem; + } + + size = qp->attrs.rq_size << RQE_SHIFT; + attr.len = size; + ret = erdma_mem_init(dev, &kqp->rq_mem, &attr); + if (ret) + goto err_free_sq_dbrec; + + kqp->rq_dbrec = + dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); + if (!kqp->rq_dbrec) { + ret = -ENOMEM; + goto err_free_rq_mem; + } + + return 0; + +err_free_rq_mem: + erdma_mem_uninit(dev, &kqp->rq_mem); +err_free_sq_dbrec: + dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma); +err_free_sq_mem: + erdma_mem_uninit(dev, &kqp->sq_mem); +err_free_rwr_tbl: + vfree(kqp->rwr_tbl); +err_free_swr_tbl: + vfree(kqp->swr_tbl); + + return ret; +} + static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, u64 dbrecords_va, struct erdma_user_dbrecords_page **dbr_page, From e43913aabd35eef837edef29e93f15e15bdeeacd Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Fri, 18 Sep 2026 17:23:37 +0800 Subject: [PATCH 053/153] RDMA/mlx5: Use DEFINE_RAW_FLEX for leftovers flow attributes Commit e91fb8b9d0ed ("RDMA/mlx5: Avoid flexible array warning") split the leftovers_specs[] array into two separate structs to silence the sparse "array of flexible structures" warning, but in doing so reversed the member order: eth_flow was placed before flow_attr. _create_flow_rule() locates the flow specs with: ib_flow = (void *)flow_attr + sizeof(*flow_attr); which expects the spec data to immediately follow the ib_flow_attr. With the reversed order, flow_attr is the last member, so ib_flow points past the end of the struct and the num_of_specs = 1 loop reads out-of-bounds memory (whatever static data follows in the .data section), producing incorrect flow rules or a crash. Rebuild the leftovers attributes with DEFINE_RAW_FLEX(), which declares an on-stack struct ib_flow_attr with a properly sized trailing flows[] array. The ETH spec lives in flows[0].eth at exactly the offset _create_flow_rule() expects, so no layout-dependent composite struct is needed. This also avoids embedding a structure that contains a flexible array member as a non-last member, which would trigger -Wflex-array-member-not-at-end once the warning is enabled. Fixes: e91fb8b9d0ed ("RDMA/mlx5: Avoid flexible array warning") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260918092337.2338-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/fs.c | 41 +++++++++++++++------------------ 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c index 98cdc5c57155..e63fdb4bc3e3 100644 --- a/drivers/infiniband/hw/mlx5/fs.c +++ b/drivers/infiniband/hw/mlx5/fs.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "mlx5_ib.h" #include "counters.h" @@ -1647,36 +1648,32 @@ static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *de struct ib_flow_attr *flow_attr, struct mlx5_flow_destination *dst) { + DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_wc, flows, 1); + DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_uc, flows, 1); + struct mlx5_ib_flow_handler *handler_ucast = NULL; struct mlx5_ib_flow_handler *handler = NULL; - static struct { - struct ib_flow_spec_eth eth_flow; - struct ib_flow_attr flow_attr; - } leftovers_wc = { .flow_attr = { .num_of_specs = 1, - .size = sizeof(leftovers_wc) }, - .eth_flow = { - .type = IB_FLOW_SPEC_ETH, - .size = sizeof(struct ib_flow_spec_eth), - .mask = { .dst_mac = { 0x1 } }, - .val = { .dst_mac = { 0x1 } } } }; + flow_attr_wc->num_of_specs = 1; + flow_attr_wc->size = sizeof(struct ib_flow_attr) + + sizeof(struct ib_flow_spec_eth); + flow_attr_wc->flows[0].eth.type = IB_FLOW_SPEC_ETH; + flow_attr_wc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth); + flow_attr_wc->flows[0].eth.mask.dst_mac[0] = 0x1; + flow_attr_wc->flows[0].eth.val.dst_mac[0] = 0x1; - static struct { - struct ib_flow_spec_eth eth_flow; - struct ib_flow_attr flow_attr; - } leftovers_uc = { .flow_attr = { .num_of_specs = 1, - .size = sizeof(leftovers_uc) }, - .eth_flow = { - .type = IB_FLOW_SPEC_ETH, - .size = sizeof(struct ib_flow_spec_eth), - .mask = { .dst_mac = { 0x1 } }, - .val = { .dst_mac = {} } } }; + flow_attr_uc->num_of_specs = 1; + flow_attr_uc->size = sizeof(struct ib_flow_attr) + + sizeof(struct ib_flow_spec_eth); + flow_attr_uc->flows[0].eth.type = IB_FLOW_SPEC_ETH; + flow_attr_uc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth); + flow_attr_uc->flows[0].eth.mask.dst_mac[0] = 0x1; - handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst); + handler = create_flow_rule(dev, ft_prio, flow_attr_wc, dst); if (!IS_ERR(handler) && flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { handler_ucast = create_flow_rule(dev, ft_prio, - &leftovers_uc.flow_attr, dst); + flow_attr_uc, dst); if (IS_ERR(handler_ucast)) { mlx5_del_flow_rules(handler->rule); ft_prio->refcount--; From c73a5f20bc786df0be484af29aa5956e909dc862 Mon Sep 17 00:00:00 2001 From: Abhijit Gangurde Date: Fri, 18 Sep 2026 15:57:13 +0530 Subject: [PATCH 054/153] net: ionic: Fetch qid allocation and SRQ capability from firmware Read the qid allocation and SRQ capability from the RDMA LIF identity reported by firmware. Signed-off-by: Abhijit Gangurde --- .../net/ethernet/pensando/ionic/ionic_if.h | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h index 15f2bc3f4043..e911fad545d6 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_if.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h @@ -7,6 +7,7 @@ #define IONIC_DEV_INFO_SIGNATURE 0x44455649 /* 'DEVI' */ #define IONIC_DEV_INFO_VERSION 1 #define IONIC_IFNAMSIZ 16 +#define IONIC_CREATE_CQ_CMD_V2_MAGIC -1 /* * enum ionic_cmd_opcode - Device commands @@ -513,6 +514,23 @@ enum ionic_lif_rdma_cap_stats { IONIC_LIF_RDMA_STAT_QP = BIT(1), }; +/** + * enum ionic_lif_rdma_alloc_qid - RDMA QID allocation capability + * + * Bitmask of RDMA queue types for which firmware can automatically + * allocate queue IDs. When a bit is set, the driver can request the + * firmware to allocate QIDs for that queue type during creation. + * + * @IONIC_LIF_RDMA_ALLOC_QID_CQ: Firmware can allocate QIDs for + * Completion Queues + * @IONIC_LIF_RDMA_ALLOC_QID_SRQ: Firmware can allocate QIDs for + * Shared Receive Queues + */ +enum ionic_lif_rdma_alloc_qid { + IONIC_LIF_RDMA_ALLOC_QID_CQ = BIT(0), + IONIC_LIF_RDMA_ALLOC_QID_SRQ = BIT(1), +}; + /** * struct ionic_lif_identity - LIF identity information (type-specific) * @@ -557,6 +575,10 @@ enum ionic_lif_rdma_cap_stats { * (enum ionic_lif_rdma_cap_stats) * @rdma.rsvd: Reserved byte * @rdma.rcq_sign_bit: RCQ sign bit + * @rdma.srq_qtype: RDMA Shared Receive Qtype + * @rdma.rsvd2: Reserved byte(s) + * @rdma.alloc_qid_cap: RDMA queue type QID allocation capability + * (bitmask of enum ionic_lif_rdma_alloc_qid) * @rdma.rsvd1: Reserved byte(s) * @words: word access to struct contents */ @@ -604,7 +626,10 @@ union ionic_lif_identity { __le16 stats_type; u8 rsvd; u8 rcq_sign_bit; - u8 rsvd1[160]; + struct ionic_lif_logical_qtype srq_qtype; + u8 rsvd2[5]; + u8 alloc_qid_cap; + u8 rsvd1[142]; } __packed rdma; } __packed; __le32 words[478]; @@ -2608,6 +2633,7 @@ struct ionic_rdma_reset_cmd { * @depth_log2: log base two of queue depth * @stride_log2: log base two of queue stride * @dma_addr: address of the queue memory + * @udma_idx: udma index * @rsvd2: reserved byte(s) * * The same command struct is used to create an RDMA event queue, completion @@ -2637,7 +2663,8 @@ struct ionic_rdma_queue_cmd { u8 depth_log2; u8 stride_log2; __le64 dma_addr; - u8 rsvd2[40]; + u8 udma_idx; + u8 rsvd2[39]; }; /****************************************************************** From 952f18e04fa9790e347e09263c25318afe723d86 Mon Sep 17 00:00:00 2001 From: Serhat Kumral Date: Fri, 18 Sep 2026 14:04:28 +0300 Subject: [PATCH 055/153] RDMA/irdma: Free IRQ when CEQ vector mapping fails irdma_cfg_ceq_vector() registers an IRQ handler before requesting the CEQ-to-vector mapping through the virtual channel. If the mapping fails, the function returns without releasing the IRQ resources. The caller destroys the CEQ, leaving the IRQ handler registered and potentially referencing freed memory. Call irdma_destroy_irq() on mapping failure to release the IRQ resources and drain the associated tasklet before the caller destroys the CEQ. Use the same dev_id passed to request_irq(), accounting for CEQ0 sharing its interrupt vector with the AEQ. Fixes: b800e82feba7 ("RDMA/irdma: Add GEN3 support for AEQ and CEQ") Assisted-by: Claude:claude-opus-5 Signed-off-by: Serhat Kumral Link: https://patch.msgid.link/20260918110428.36586-1-serhatkumral1@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/hw.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index cae4f316def4..e28f5f4876cd 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -1252,12 +1252,21 @@ static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, } msix_vec->ceq_id = ceq_id; - if (rf->sc_dev.privileged) + if (rf->sc_dev.privileged) { rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx, true); - else + } else { status = irdma_vchnl_req_ceq_vec_map(&rf->sc_dev, ceq_id, msix_vec->idx); + if (!status) + return 0; + + if (rf->msix_shared && !ceq_id) + irdma_destroy_irq(rf, msix_vec, rf); + else + irdma_destroy_irq(rf, msix_vec, iwceq); + } + return status; } From ba3472ed6718bf3a681e7ab755db33cf284becd5 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:19 +0300 Subject: [PATCH 056/153] RDMA/erdma: Pin CQ buffer writable to match device DMA write access erdma_init_user_cq() pins the CQ ring with access=0, but the device writes CQEs into it. Pass IB_ACCESS_LOCAL_WRITE so the buffer is pinned writable and marked dirty on unpin. Fixes: 155055771704 ("RDMA/erdma: Add verbs implementation") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-2-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/erdma/erdma_verbs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 80d61430d2a4..7d76d7793292 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -1999,6 +1999,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, attr.virt = ureq->qbuf_va; attr.len = ureq->qbuf_len; attr.req_page_size = SZ_64M - SZ_4K; + attr.access = IB_ACCESS_LOCAL_WRITE; ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr); if (ret) return ret; From ce29f6e741844a78ad1977f2a550ede27ecf95f8 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:20 +0300 Subject: [PATCH 057/153] RDMA/hns: Pin CQ buffer writable to match device DMA write access alloc_cq_buf() leaves user_access zero-initialized, but the device writes CQEs into the CQ buffer. Set IB_ACCESS_LOCAL_WRITE so the buffer is pinned writable, matching the device's write access. Fixes: 744b7bdfa79e ("RDMA/hns: Support 0 hop addressing for CQE buffer") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-3-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_cq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 1dd0efb5620d..ae314f136732 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -261,6 +261,7 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size; buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num; buf_attr.region_count = 1; + buf_attr.user_access = IB_ACCESS_LOCAL_WRITE; ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr, hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT, From 89abac4b60b708037595255ea2614b6967c86ee5 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:21 +0300 Subject: [PATCH 058/153] RDMA/vmw_pvrdma: Pin QP and SRQ rings writable to match device DMA write access pvrdma embeds a pvrdma_ring_state header directly inside the QP and SRQ ring buffers, and the hypervisor writes cons_head into that header. Pinning with access=0 does not force a private page copy (no FOLL_WRITE), so the hypervisor may write into a shared page. Pass IB_ACCESS_LOCAL_WRITE for all three rings. The SRQ ring_state is not currently used by the driver but is pinned writable for consistency. Fixes: 29c8d9eba550 ("IB: Add vmw_pvrdma driver") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-4-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c | 6 ++++-- drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index e939cd5ce40b..53cc49f2b8c9 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -270,7 +270,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, /* set qp->sq.wqe_cnt, shift, buf_size.. */ qp->rumem = ib_umem_get_va(ibqp->device, ucmd.rbuf_addr, - ucmd.rbuf_size, 0); + ucmd.rbuf_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(qp->rumem)) { ret = PTR_ERR(qp->rumem); goto err_qp; @@ -282,7 +283,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, } qp->sumem = ib_umem_get_va(ibqp->device, ucmd.sbuf_addr, - ucmd.sbuf_size, 0); + ucmd.sbuf_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(qp->sumem)) { if (!is_srq) ib_umem_release(qp->rumem); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c index 345ec486a223..3252c2eb405a 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c @@ -146,7 +146,8 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, if (ret) goto err_srq; - srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0); + srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(srq->umem)) { ret = PTR_ERR(srq->umem); goto err_srq; From b577999f6f8dbf954bb915a224a69c57c4476844 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:22 +0300 Subject: [PATCH 059/153] RDMA/umem: Reuse ib_umem_get_cq_buf_or_va() for VA-only CQ pinning Several drivers pin their CQ ring buffer via ib_umem_get_va() rather than the CQ-specific helper. Switch them to ib_umem_get_cq_buf_or_va() so the subsequent DMA_FROM_DEVICE change covers all CQ paths at once. For shared helpers used by both CQ and non-CQ callers (qedr, mana, erdma, hns) a new is_cq parameter selects the appropriate pinning function; non-CQ callers pass false. vmw_pvrdma's CQ is excluded: it embeds a ring-state header that the driver CPU-writes after polling, making it genuinely bidirectional. This is a pure refactor with no functional change. Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-5-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 9 ++++---- drivers/infiniband/hw/erdma/erdma_verbs.c | 11 +++++++-- drivers/infiniband/hw/erdma/erdma_verbs.h | 1 + drivers/infiniband/hw/hns/hns_roce_cq.c | 2 +- drivers/infiniband/hw/hns/hns_roce_device.h | 2 +- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 2 +- drivers/infiniband/hw/hns/hns_roce_mr.c | 22 +++++++++++++----- drivers/infiniband/hw/hns/hns_roce_qp.c | 2 +- drivers/infiniband/hw/hns/hns_roce_srq.c | 4 ++-- .../infiniband/hw/ionic/ionic_controlpath.c | 5 ++-- drivers/infiniband/hw/mana/cq.c | 2 +- drivers/infiniband/hw/mana/main.c | 9 ++++++-- drivers/infiniband/hw/mana/mana_ib.h | 2 +- drivers/infiniband/hw/mana/qp.c | 9 ++++---- drivers/infiniband/hw/mana/wq.c | 3 ++- drivers/infiniband/hw/mlx4/cq.c | 14 ++++++----- drivers/infiniband/hw/mlx5/cq.c | 6 ++--- drivers/infiniband/hw/qedr/verbs.c | 23 +++++++++++++------ 18 files changed, 83 insertions(+), 45 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index ef08d42f377e..7f9aa3620abb 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -3749,12 +3749,13 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, if (rc) goto fail; - cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, - entries * sizeof(struct cq_base), - IB_ACCESS_LOCAL_WRITE); + cq->resize_umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, NULL, + req.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->resize_umem)) { rc = PTR_ERR(cq->resize_umem); - ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n", + ibdev_err(&rdev->ibdev, "%s: ib_umem_get_cq_buf_or_va failed! rc = %pe\n", __func__, cq->resize_umem); cq->resize_umem = NULL; goto fail; diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 7d76d7793292..d114cc5a0c7c 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -788,8 +788,14 @@ static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem, switch (mem->type) { case ERDMA_UMEM: - mem->umem = ib_umem_get_va(&dev->ibdev, attr->start, attr->len, - attr->access); + if (attr->flags & ERDMA_MEM_FLAG_CQ_BUF) + mem->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, + attr->start, + attr->len, + attr->access); + else + mem->umem = ib_umem_get_va(&dev->ibdev, attr->start, + attr->len, attr->access); if (IS_ERR(mem->umem)) { ret = PTR_ERR(mem->umem); return ret; @@ -2000,6 +2006,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, attr.len = ureq->qbuf_len; attr.req_page_size = SZ_64M - SZ_4K; attr.access = IB_ACCESS_LOCAL_WRITE; + attr.flags = ERDMA_MEM_FLAG_CQ_BUF; ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr); if (ret) return ret; diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index 02bf3723680d..0542e10a0c99 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -128,6 +128,7 @@ struct erdma_kmem { enum erdma_mem_flags { ERDMA_MEM_FLAG_MR_BUF = (1 << 0), + ERDMA_MEM_FLAG_CQ_BUF = (1 << 1), }; struct erdma_mem_init_attr { diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index ae314f136732..7dfaa01cda67 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -265,7 +265,7 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr, hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT, - udata, addr); + udata, addr, true); if (ret) ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 4a1c2fd4c187..5890ee69247c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -1232,7 +1232,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, struct hns_roce_buf_attr *buf_attr, unsigned int page_shift, struct ib_udata *udata, - unsigned long user_addr); + unsigned long user_addr, bool is_cq); void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr); int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 35be5b038f68..a8f319a57f25 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -6940,7 +6940,7 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr, hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL, - 0); + 0, false); if (err) dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 7d41ae897458..4799f667eeda 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -112,7 +112,7 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr, hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT, - udata, start); + udata, start, false); if (err) { ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err); return err; @@ -586,7 +586,8 @@ static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, struct hns_roce_buf_attr *buf_attr, - struct ib_udata *udata, unsigned long user_addr) + struct ib_udata *udata, unsigned long user_addr, + bool is_cq) { struct ib_device *ibdev = &hr_dev->ib_dev; size_t total_size; @@ -595,8 +596,14 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, if (udata) { mtr->kmem = NULL; - mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size, - buf_attr->user_access); + if (is_cq) + mtr->umem = ib_umem_get_cq_buf_or_va(ibdev, NULL, + user_addr, + total_size, + buf_attr->user_access); + else + mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size, + buf_attr->user_access); if (IS_ERR(mtr->umem)) { ibdev_err(ibdev, "failed to get umem, ret = %pe.\n", mtr->umem); @@ -1035,11 +1042,13 @@ static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) * @ba_page_shift: page shift for multi-hop base address table * @udata: user space context, if it's NULL, means kernel space * @user_addr: userspace virtual address to start at + * @is_cq: true when @mtr backs a CQ buffer, which the device writes + * completion entries into */ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, struct hns_roce_buf_attr *buf_attr, unsigned int ba_page_shift, struct ib_udata *udata, - unsigned long user_addr) + unsigned long user_addr, bool is_cq) { struct ib_device *ibdev = &hr_dev->ib_dev; int ret; @@ -1052,7 +1061,8 @@ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, mtr->umem = NULL; mtr->kmem = NULL; } else { - ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr); + ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr, + is_cq); if (ret) { ibdev_err(ibdev, "failed to alloc mtr bufs, ret = %d.\n", ret); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index e333a8c4acb5..c6c0dfdbdbc3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -808,7 +808,7 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, } ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, - udata, addr); + udata, addr, false); if (ret) { ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); goto err_inline; diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 4a54394f96be..b5e9ac9cfd59 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -180,7 +180,7 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr, hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT, - udata, addr); + udata, addr, false); if (ret) { ibdev_err(ibdev, "failed to alloc SRQ idx mtr, ret = %d.\n", ret); @@ -235,7 +235,7 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev, ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr, hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT, - udata, addr); + udata, addr, false); if (ret) ibdev_err(ibdev, "failed to alloc SRQ buf mtr, ret = %d.\n", ret); diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index 37f71fb43811..f392c635338b 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -110,8 +110,9 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, if (rc) goto err_qdesc; - cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr, - req_cq->size, IB_ACCESS_LOCAL_WRITE); + cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, + req_cq->addr, req_cq->size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { rc = PTR_ERR(cq->umem); goto err_qdesc; diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 6f9ac8b4aac8..de81a4b994e0 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -40,7 +40,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->cqe = attr->cqe; err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, - &cq->queue); + &cq->queue, true); if (err) { ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); return err; diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index 672bd3f650dd..858bbc325eb2 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -295,7 +295,7 @@ int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_qu } int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, - struct mana_ib_queue *queue) + struct mana_ib_queue *queue, bool is_cq) { struct ib_umem *umem; int err; @@ -304,7 +304,12 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, queue->id = INVALID_QUEUE_ID; queue->gdma_region = GDMA_INVALID_DMA_REGION; - umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); + if (is_cq) + umem = ib_umem_get_cq_buf_or_va(&mdev->ib_dev, NULL, addr, + size, IB_ACCESS_LOCAL_WRITE); + else + umem = ib_umem_get_va(&mdev->ib_dev, addr, size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem); return PTR_ERR(umem); diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 4da110b302c7..8d63bc167e42 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -731,7 +731,7 @@ int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type, struct mana_ib_queue *queue); int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, - struct mana_ib_queue *queue); + struct mana_ib_queue *queue, bool is_cq); void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue); struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index fac43b3a5eb7..88e792cb0f85 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -340,7 +340,8 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", ucmd.sq_buf_addr, ucmd.port); - err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq); + err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq, + false); if (err) { ibdev_dbg(&mdev->ib_dev, "Failed to create queue for create qp-raw, err %d\n", err); @@ -579,7 +580,7 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE) { flags &= ~MANA_RC_FLAG_NO_MMQ; err = mana_ib_create_queue(mdev, ucmd.mmq_buf, ucmd.mmq_size, - &qp->rc_qp.queues[i]); + &qp->rc_qp.queues[i], false); if (err) goto destroy_queues; } else { @@ -589,7 +590,7 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, continue; } err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j], - &qp->rc_qp.queues[i]); + &qp->rc_qp.queues[i], false); if (err) goto destroy_queues; j++; @@ -663,7 +664,7 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; ++i) { err = mana_ib_create_queue(mdev, ucmd.queue_buf[i], ucmd.queue_size[i], - &qp->uc_qp.queues[i]); + &qp->uc_qp.queues[i], false); if (err) goto destroy_queues; } diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c index 6b066d605dcb..4042f26632c0 100644 --- a/drivers/infiniband/hw/mana/wq.c +++ b/drivers/infiniband/hw/mana/wq.c @@ -25,7 +25,8 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr); - err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue); + err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue, + false); if (err) { ibdev_dbg(&mdev->ib_dev, "Failed to create queue for create wq, %d\n", err); diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 98e3eb5cf96e..81dc47e57352 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -185,9 +185,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, goto err_umem; } } else { - cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); + cq->umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, + ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { err = PTR_ERR(cq->umem); goto err_cq; @@ -354,9 +355,10 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq if (!cq->resize_buf) return -ENOMEM; - cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); + cq->resize_umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, + ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->resize_umem)) { err = PTR_ERR(cq->resize_umem); goto err_buf; diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 51892911e32a..072b77a2c8a6 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -1246,9 +1246,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1) return -EINVAL; - umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, - (size_t)ucmd.cqe_size * entries, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, ucmd.buf_addr, + (size_t)ucmd.cqe_size * entries, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { err = PTR_ERR(umem); return err; diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 973d0fa7cdce..acab3cb98caa 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -775,16 +775,23 @@ static inline int qedr_init_user_queue(struct ib_udata *udata, struct qedr_userq *q, u64 buf_addr, size_t buf_len, bool requires_db_rec, int access, - int alloc_and_init) + int alloc_and_init, bool is_cq) { u32 fw_pages; int rc; q->buf_addr = buf_addr; q->buf_len = buf_len; - q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, access); + if (is_cq) + q->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, + q->buf_addr, q->buf_len, + access); + else + q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, + access); if (IS_ERR(q->umem)) { - DP_ERR(dev, "create user queue: failed ib_umem_get_va, got %ld\n", + DP_ERR(dev, "create user queue: failed %s, got %ld\n", + is_cq ? "ib_umem_get_cq_buf_or_va" : "ib_umem_get_va", PTR_ERR(q->umem)); return PTR_ERR(q->umem); } @@ -946,7 +953,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, ureq.len, true, IB_ACCESS_LOCAL_WRITE, - 1); + 1, true); if (rc) goto err0; @@ -1438,7 +1445,7 @@ static int qedr_init_srq_user_params(struct ib_udata *udata, int rc; rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, - ureq->srq_len, false, access, 1); + ureq->srq_len, false, access, 1, false); if (rc) return rc; @@ -1831,7 +1838,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev, if (qedr_qp_has_sq(qp)) { /* SQ - read access only (0) */ rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, - ureq.sq_len, true, 0, alloc_and_init); + ureq.sq_len, true, 0, alloc_and_init, + false); if (rc) return rc; } @@ -1839,7 +1847,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev, if (qedr_qp_has_rq(qp)) { /* RQ - read access only (0) */ rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, - ureq.rq_len, true, 0, alloc_and_init); + ureq.rq_len, true, 0, alloc_and_init, + false); if (rc) { ib_umem_release(qp->usq.umem); qp->usq.umem = NULL; From 2445d74c465ac0200f6464b24cb5eb76319c1be1 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:23 +0300 Subject: [PATCH 060/153] RDMA/umem: Support an explicit DMA direction other than DMA_BIDIRECTIONAL Add a dma_dir field to struct ib_umem to record the direction chosen at map time and reuse it at unmap. Thread an explicit dma_data_direction parameter through the internal pinning helpers so each caller can supply the direction that matches the device's actual access pattern. All existing callers still pass DMA_BIDIRECTIONAL, so this is a pure mechanism addition with no behavioural change. Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-6-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/umem.c | 64 ++++++++++++++++++++++------------ include/rdma/ib_umem.h | 2 ++ 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 88110b9661f5..c2f277ada042 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -55,7 +55,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d if (dirty) ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt, - DMA_BIDIRECTIONAL, umem->dma_attrs); + umem->dma_dir, umem->dma_attrs); for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) { unpin_user_page_range_dirty_lock(sg_page(sg), @@ -161,7 +161,8 @@ EXPORT_SYMBOL(ib_umem_find_best_pgsz); static struct ib_umem *__ib_umem_get_va(struct ib_device *device, unsigned long addr, size_t size, - int access) + int access, + enum dma_data_direction dir) { struct ib_umem *umem; struct page **page_list; @@ -202,6 +203,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, */ umem->iova = addr; umem->writable = ib_access_writable(access); + umem->dma_dir = dir; umem->owning_mm = mm = current->mm; umem->dma_attrs = DMA_ATTR_REQUIRE_COHERENT; if (access & IB_ACCESS_RELAXED_ORDERING) @@ -261,7 +263,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, } ret = ib_dma_map_sgtable_attrs(device, &umem->sgt_append.sgt, - DMA_BIDIRECTIONAL, umem->dma_attrs); + dir, umem->dma_attrs); if (ret) goto umem_release; goto out; @@ -279,17 +281,16 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, return ret ? ERR_PTR(ret) : umem; } -/** - * ib_umem_get_desc - Pin a umem from a buffer descriptor. - * @device: IB device. - * @desc: buffer descriptor (VA or DMABUF). - * @access: IB access flags. +/* + * __ib_umem_get_desc_dir - core implementation for ib_umem_get_desc(). * - * Return: caller-owned umem on success, ERR_PTR(...) on error. + * @dir applies to VA buffers only; dmabuf direction is managed by the + * dmabuf subsystem and this argument is ignored for that type. */ -struct ib_umem *ib_umem_get_desc(struct ib_device *device, - const struct ib_uverbs_buffer_desc *desc, - int access) +static struct ib_umem * +__ib_umem_get_desc_dir(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + int access, enum dma_data_direction dir) { struct ib_umem_dmabuf *umem_dmabuf; @@ -310,11 +311,26 @@ struct ib_umem *ib_umem_get_desc(struct ib_device *device, return &umem_dmabuf->umem; case IB_UVERBS_BUFFER_TYPE_VA: return __ib_umem_get_va(device, desc->addr, desc->length, - access); + access, dir); default: return ERR_PTR(-EINVAL); } } + +/** + * ib_umem_get_desc - Pin a umem from a buffer descriptor. + * @device: IB device. + * @desc: buffer descriptor (VA or DMABUF). + * @access: IB access flags. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_desc(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + int access) +{ + return __ib_umem_get_desc_dir(device, desc, access, DMA_BIDIRECTIONAL); +} EXPORT_SYMBOL(ib_umem_get_desc); /* @@ -376,11 +392,12 @@ static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs, static struct ib_umem * ib_umem_get_desc_check(struct ib_device *device, const struct ib_uverbs_buffer_desc *desc, - size_t min_size, int access) + size_t min_size, int access, + enum dma_data_direction dir) { struct ib_umem *umem; - umem = ib_umem_get_desc(device, desc, access); + umem = __ib_umem_get_desc_dir(device, desc, access, dir); if (IS_ERR(umem)) return umem; if (umem->length < min_size) { @@ -401,7 +418,7 @@ static struct ib_umem * ib_umem_get_from_attrs(struct ib_device *device, const struct uverbs_attr_bundle *attrs, u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler, - size_t size, int access) + size_t size, int access, enum dma_data_direction dir) { struct ib_uverbs_buffer_desc desc = {}; int ret; @@ -411,7 +428,7 @@ ib_umem_get_from_attrs(struct ib_device *device, return NULL; if (ret) return ERR_PTR(ret); - return ib_umem_get_desc_check(device, &desc, size, access); + return ib_umem_get_desc_check(device, &desc, size, access, dir); } /* @@ -431,7 +448,8 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device, const struct uverbs_attr_bundle *attrs, u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler, - u64 addr, size_t size, int access) + u64 addr, size_t size, int access, + enum dma_data_direction dir) { struct ib_uverbs_buffer_desc desc = {}; int ret; @@ -445,7 +463,7 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device, }; else if (ret) return ERR_PTR(ret); - return ib_umem_get_desc_check(device, &desc, size, access); + return ib_umem_get_desc_check(device, &desc, size, access, dir); } /** @@ -464,7 +482,7 @@ struct ib_umem *ib_umem_get_attr(struct ib_device *device, u16 attr_id, size_t size, int access) { return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size, - access); + access, DMA_BIDIRECTIONAL); } EXPORT_SYMBOL(ib_umem_get_attr); @@ -503,7 +521,7 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, int access) { return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr, - size, access); + size, access, DMA_BIDIRECTIONAL); } EXPORT_SYMBOL(ib_umem_get_attr_or_va); @@ -579,7 +597,7 @@ struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, return ib_umem_get_from_attrs(device, attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, uverbs_create_cq_get_buffer_desc, - size, access); + size, access, DMA_BIDIRECTIONAL); } EXPORT_SYMBOL(ib_umem_get_cq_buf); @@ -608,7 +626,7 @@ struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, return ib_umem_get_from_attrs_or_va(device, attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, uverbs_create_cq_get_buffer_desc, - addr, size, access); + addr, size, access, DMA_BIDIRECTIONAL); } EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va); diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 1fe87fd1d769..dcb645419692 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -7,6 +7,7 @@ #ifndef IB_UMEM_H #define IB_UMEM_H +#include #include struct ib_device; @@ -19,6 +20,7 @@ struct ib_umem { size_t length; unsigned long address; unsigned long dma_attrs; + enum dma_data_direction dma_dir; u32 writable : 1; u32 is_odp : 1; u32 is_dmabuf : 1; From 50b96c3e3feb6161df4a1ac4975e885602c79bfd Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:24 +0300 Subject: [PATCH 061/153] RDMA/umem: Map CQ buffers DMA_FROM_DEVICE For a CQ ring the device writes CQEs and the CPU only reads. DMA_FROM_DEVICE is the correct direction. Use it explicitly in ib_umem_get_cq_buf() and ib_umem_get_cq_buf_or_va() rather than deriving it from the access flags: CQ callers pass IB_ACCESS_LOCAL_WRITE so that get_user_pages() pins a writable private page (FOLL_WRITE), which the device needs to DMA-write CQEs into. That flag would yield DMA_BIDIRECTIONAL from the access-flag derivation, which is wrong for the bus direction. Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-7-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/umem.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index c2f277ada042..f1da70c4e352 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -587,6 +587,9 @@ static int uverbs_create_cq_get_buffer_desc(const struct uverbs_attr_bundle *att * must arrange its own backing (typically an in-kernel allocation) * when no source is available. * + * The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it + * and the CPU only reads. + * * Return: caller-owned umem on success; NULL when no source supplied * a buffer; ERR_PTR(...) on error. */ @@ -597,7 +600,7 @@ struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, return ib_umem_get_from_attrs(device, attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, uverbs_create_cq_get_buffer_desc, - size, access, DMA_BIDIRECTIONAL); + size, access, DMA_FROM_DEVICE); } EXPORT_SYMBOL(ib_umem_get_cq_buf); @@ -613,6 +616,9 @@ EXPORT_SYMBOL(ib_umem_get_cq_buf); * Like ib_umem_get_cq_buf(), but pins @addr/@size when neither the * UMEM attribute nor the legacy CQ buffer attributes are supplied. * + * The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it + * and the CPU only reads. + * * See ib_umem_get_attr_or_va() for the note on @size's dual role and * the migration path for drivers that would distinguish a user-supplied * length from a driver-computed minimum. @@ -626,7 +632,7 @@ struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, return ib_umem_get_from_attrs_or_va(device, attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, uverbs_create_cq_get_buffer_desc, - addr, size, access, DMA_BIDIRECTIONAL); + addr, size, access, DMA_FROM_DEVICE); } EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va); From dfc5334af5667e2ef24c54ec9ce88c4eb1cdba64 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:25 +0300 Subject: [PATCH 062/153] RDMA/umem: Derive DMA direction from IB access flags Map non-writable user memory DMA_TO_DEVICE: when no write access is granted, the NIC only reads the pages and DMA_BIDIRECTIONAL is unnecessarily broad. Where the IOMMU enforces direction this prevents the device writing to read-only buffers; where it does not the mapping is still semantically correct. Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-8-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/umem.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index f1da70c4e352..b1ab4f0359b0 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -159,6 +159,11 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, } EXPORT_SYMBOL(ib_umem_find_best_pgsz); +static inline enum dma_data_direction ib_access_dma_dir(int access) +{ + return ib_access_writable(access) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; +} + static struct ib_umem *__ib_umem_get_va(struct ib_device *device, unsigned long addr, size_t size, int access, @@ -329,7 +334,8 @@ struct ib_umem *ib_umem_get_desc(struct ib_device *device, const struct ib_uverbs_buffer_desc *desc, int access) { - return __ib_umem_get_desc_dir(device, desc, access, DMA_BIDIRECTIONAL); + return __ib_umem_get_desc_dir(device, desc, access, + ib_access_dma_dir(access)); } EXPORT_SYMBOL(ib_umem_get_desc); @@ -482,7 +488,7 @@ struct ib_umem *ib_umem_get_attr(struct ib_device *device, u16 attr_id, size_t size, int access) { return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size, - access, DMA_BIDIRECTIONAL); + access, ib_access_dma_dir(access)); } EXPORT_SYMBOL(ib_umem_get_attr); @@ -521,7 +527,8 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, int access) { return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr, - size, access, DMA_BIDIRECTIONAL); + size, access, + ib_access_dma_dir(access)); } EXPORT_SYMBOL(ib_umem_get_attr_or_va); From 4354ab7a85d1ddd45a8a3337227fad5b37b44902 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:26 +0300 Subject: [PATCH 063/153] RDMA/mlx5: Fix mlx5_ib_dev_res_init() failure when XRC cap is absent mlx5_ib_dev_res_init() returns -EOPNOTSUPP when the firmware XRC capability is absent, failing driver probe on XRC-less devices. Make XRC optional throughout the devr resource path: initialize cq_lock and srq_lock unconditionally (needed for every QP creation), skip xrcd allocation and the XRC-type placeholder SRQ (s0) when xrc=0, and guard their cleanup paths accordingly. Fixes: f4375443b786 ("RDMA/mlx5: Get XRCD number directly for the internal use") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-9-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/main.c | 54 +++++++++++++++++++------------ 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 373ee1f42d4a..8b8a0f26cf19 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3373,7 +3373,7 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev) { struct mlx5_ib_resources *devr = &dev->devr; struct ib_srq_init_attr attr; - struct ib_srq *s0, *s1; + struct ib_srq *s0 = NULL, *s1; int ret = 0; /* @@ -3391,19 +3391,27 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev) if (ret) goto unlock; - memset(&attr, 0, sizeof(attr)); - attr.attr.max_sge = 1; - attr.attr.max_wr = 1; - attr.srq_type = IB_SRQT_XRC; - attr.ext.cq = devr->c0; + /* + * s0 is an XRC-type placeholder SRQ used as the default XRQN for + * XRC QPs. Skip it when XRC is absent; all devr->s0 accesses in + * qp.c are inside XRC QP paths that the verbs layer blocks before + * reaching this driver when xrc=0, so NULL is safe. + */ + if (MLX5_CAP_GEN(dev->mdev, xrc)) { + memset(&attr, 0, sizeof(attr)); + attr.attr.max_sge = 1; + attr.attr.max_wr = 1; + attr.srq_type = IB_SRQT_XRC; + attr.ext.cq = devr->c0; - s0 = ib_create_srq(devr->p0, &attr); - if (IS_ERR(s0)) { - ret = PTR_ERR(s0); - mlx5_ib_err(dev, - "Couldn't create SRQ 0 for res init, err=%pe\n", - s0); - goto unlock; + s0 = ib_create_srq(devr->p0, &attr); + if (IS_ERR(s0)) { + ret = PTR_ERR(s0); + mlx5_ib_err(dev, + "Couldn't create SRQ 0 for res init, err=%pe\n", + s0); + goto unlock; + } } memset(&attr, 0, sizeof(attr)); @@ -3417,7 +3425,8 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev) mlx5_ib_err(dev, "Couldn't create SRQ 1 for res init, err=%pe\n", s1); - ib_destroy_srq(s0); + if (s0) + ib_destroy_srq(s0); goto unlock; } @@ -3434,8 +3443,11 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) struct mlx5_ib_resources *devr = &dev->devr; int ret; + mutex_init(&devr->cq_lock); + mutex_init(&devr->srq_lock); + if (!MLX5_CAP_GEN(dev->mdev, xrc)) - return -EOPNOTSUPP; + return 0; ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0); if (ret) @@ -3447,9 +3459,6 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) return ret; } - mutex_init(&devr->cq_lock); - mutex_init(&devr->srq_lock); - return 0; } @@ -3460,10 +3469,13 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev) /* After s0/s1 init, they are not unset during the device lifetime. */ if (devr->s1) { ib_destroy_srq(devr->s1); - ib_destroy_srq(devr->s0); + if (devr->s0) + ib_destroy_srq(devr->s0); + } + if (MLX5_CAP_GEN(dev->mdev, xrc)) { + mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0); + mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0); } - mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0); - mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0); /* After p0/c0 init, they are not unset during the device lifetime. */ if (devr->c0) { ib_destroy_cq(devr->c0); From 28c559d1e25de46bb6dd39e6fa1e93ea6abdf1f6 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:28 +0300 Subject: [PATCH 064/153] RDMA/mlx5: Set WQ event handler before firmware RQ insertion create_rq() inserts the RQ into dev->qp_table.tree, making it visible to rsc_event_notifier() before rwq->core_qp.event was assigned. A firmware WQ_CATAS_ERROR event in that window finds a NULL event pointer. Move the assignment before create_rq(). Fixes: 350d0e4c7e4b ("IB/mlx5: Track asynchronous events on a receive work queue") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-11-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index b1bda6e5458b..b367139a1f30 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -5619,6 +5619,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, rwq = kzalloc_obj(*rwq); if (!rwq) return ERR_PTR(-ENOMEM); + rwq->core_qp.event = mlx5_ib_wq_event; err = prepare_user_rq(pd, init_attr, udata, rwq); if (err) goto err; @@ -5642,7 +5643,6 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, goto err_copy; } - rwq->core_qp.event = mlx5_ib_wq_event; rwq->ibwq.event_handler = init_attr->event_handler; return &rwq->ibwq; From d2aa15620e98dc4c6516944397ebea03b7a62b90 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:29 +0300 Subject: [PATCH 065/153] RDMA/mlx5: Set SRQ event handler before xarray insertion mlx5_cmd_create_srq() stores the SRQ into the xarray before srq->msrq.event was assigned. A firmware SRQ event arriving in that window calls through a NULL pointer. Move the assignment before mlx5_cmd_create_srq(). Fixes: e126ba97dba9 ("mlx5: Add driver for Mellanox Connect-IB adapters") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-12-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/srq.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index a973c1b7515f..6a4a937d97ce 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -284,6 +284,7 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, in.pd = to_mpd(ib_srq->pd)->pdn; in.db_record = srq->db.dma; + srq->msrq.event = mlx5_ib_srq_event; err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); kvfree(in.pas); if (err) { @@ -292,8 +293,6 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, } mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); - - srq->msrq.event = mlx5_ib_srq_event; srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; if (udata) { From a4fa6497002c80b55ebe597242a1bba2ff268886 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:30 +0300 Subject: [PATCH 066/153] RDMA/mlx5: Set RQ event handler for raw-packet QP create_raw_packet_qp() creates a separate firmware RQ object but never assigned rq->base.mqp.event. Any firmware event on this resource (e.g. WQ_CATAS_ERROR, WQ_ACCESS_ERROR) calls through a NULL pointer. Set the event pointer before the RQ is inserted into the resource table. Fixes: 0fb2ed66a14c ("IB/mlx5: Add create and destroy functionality for Raw Packet QP") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-13-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index b367139a1f30..38f1c03ae586 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -1647,6 +1647,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, } rq->base.container_mibqp = qp; + rq->base.mqp.event = mlx5_ib_qp_event; if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING) rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING; From 05a078c97eb0bb7158526dae414b853737692485 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:31 +0300 Subject: [PATCH 067/153] RDMA/mlx5: Set QP event handler before firmware QPC insertion Four QP creation paths assigned base->container_mibqp and base->mqp.event after the firmware create call that inserts the QP into dev->qp_table.tree. A firmware error EQE in that window reaches qp->event() with a NULL pointer. Move both assignments before the firmware call. Also set ibqp.qp_num inside mlx5_qpc_create_qp() before create_resource_common() so an EQE arriving early does not observe qp_num==0. In addition, add a WARN_ON_ONCE(!qp->event) guard in rsc_event_notifier() as a safeguard for future regressions. Fixes: e126ba97dba9 ("mlx5: Add driver for Mellanox Connect-IB adapters") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-14-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qp.c | 22 ++++++++++------------ drivers/infiniband/hw/mlx5/qpc.c | 10 ++++++++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 38f1c03ae586..627dd17804c9 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -1624,6 +1624,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, if (err) return err; + sq->base.container_mibqp = qp; + sq->base.mqp.event = mlx5_ib_qp_event; err = create_raw_packet_qp_sq(dev, udata, attrs, sq, in, pd, to_mcq(init_attr->send_cq)); if (err) @@ -1635,9 +1637,6 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, resp->sqn = sq->base.mqp.qpn; resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN; } - - sq->base.container_mibqp = qp; - sq->base.mqp.event = mlx5_ib_qp_event; } if (qp->rq.wqe_cnt) { @@ -2093,13 +2092,13 @@ static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, } base = &qp->trans_qp.base; + base->container_mibqp = qp; + base->mqp.event = mlx5_ib_qp_event; err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out); kvfree(in); if (err) return err; - base->container_mibqp = qp; - base->mqp.event = mlx5_ib_qp_event; if (MLX5_CAP_GEN(mdev, ece_support)) params->resp.ece_options = MLX5_GET(create_qp_out, out, ece); @@ -2237,14 +2236,14 @@ static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING; } + base->container_mibqp = qp; + base->mqp.event = mlx5_ib_qp_event; err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out); kvfree(in); if (err) goto err_create; - base->container_mibqp = qp; - base->mqp.event = mlx5_ib_qp_event; if (MLX5_CAP_GEN(mdev, ece_support)) params->resp.ece_options = MLX5_GET(create_qp_out, out, ece); @@ -2433,6 +2432,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING; } + base->container_mibqp = qp; + base->mqp.event = mlx5_ib_qp_event; if (init_attr->qp_type == IB_QPT_RAW_PACKET || qp->flags & IB_QP_CREATE_SOURCE_QPN) { qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr; @@ -2447,8 +2448,6 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (err) goto err_create; - base->container_mibqp = qp; - base->mqp.event = mlx5_ib_qp_event; if (MLX5_CAP_GEN(mdev, ece_support)) params->resp.ece_options = MLX5_GET(create_qp_out, out, ece); @@ -2578,14 +2577,13 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, MLX5_CAP_GEN(mdev, go_back_n)) MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N); + base->container_mibqp = qp; + base->mqp.event = mlx5_ib_qp_event; err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out); kvfree(in); if (err) goto err_create; - base->container_mibqp = qp; - base->mqp.event = mlx5_ib_qp_event; - get_cqs(qp->type, attr->send_cq, attr->recv_cq, &send_cq, &recv_cq); spin_lock_irqsave(&dev->reset_flow_resource_lock, flags); diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c index a7a4f9420271..77ddab666ecd 100644 --- a/drivers/infiniband/hw/mlx5/qpc.c +++ b/drivers/infiniband/hw/mlx5/qpc.c @@ -146,6 +146,8 @@ static int rsc_event_notifier(struct notifier_block *nb, case MLX5_RES_RQ: case MLX5_RES_SQ: qp = (struct mlx5_core_qp *)common; + if (WARN_ON_ONCE(!qp->event)) + goto out; qp->event(qp, event_type); /* Need to put resource in event handler */ return NOTIFY_OK; @@ -258,6 +260,14 @@ int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp, qp->uid = MLX5_GET(create_qp_in, in, uid); qp->qpn = MLX5_GET(create_qp_out, out, qpn); + /* Set ibqp.qp_num before create_resource_common() inserts the QP into + * the radix tree and makes it visible to EQE processing. The + * assignment in create_qp() happens only after this function returns, + * leaving a window where an arriving EQE would observe qp_num == 0. + * create_qp() still overrides this with 0/1 for QP0/QP1, and sets it + * independently for DCT (which does not go through this function). + */ + to_mibqp(qp)->ibqp.qp_num = qp->qpn; err = create_resource_common(dev, qp, MLX5_RES_QP); if (err) From 829a0e65deff3f826a7870a52404aeaf1a3fec3f Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:32 +0300 Subject: [PATCH 068/153] RDMA/mlx5: Initialize QP/RQ/SQ resource refcount before publishing it create_resource_common() inserted the resource into the radix tree before its initialization was complete. A firmware EQE in that window finds a zero refcount_t and calls refcount_inc() on it; the refcount API treats this as a use-after-free, saturates the counter, and permanently breaks the resource's lifecycle tracking. Move the initialization before the radix_tree_insert(). Fixes: e126ba97dba9 ("mlx5: Add driver for Mellanox Connect-IB adapters") Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-15-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qpc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c index 77ddab666ecd..8b5bd33a7f0f 100644 --- a/drivers/infiniband/hw/mlx5/qpc.c +++ b/drivers/infiniband/hw/mlx5/qpc.c @@ -167,6 +167,10 @@ static int create_resource_common(struct mlx5_ib_dev *dev, int err; qp->common.res = rsc_type; + refcount_set(&qp->common.refcount, 1); + init_completion(&qp->common.free); + qp->pid = current->pid; + spin_lock_irq(&table->lock); err = radix_tree_insert(&table->tree, qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN), @@ -175,10 +179,6 @@ static int create_resource_common(struct mlx5_ib_dev *dev, if (err) return err; - refcount_set(&qp->common.refcount, 1); - init_completion(&qp->common.free); - qp->pid = current->pid; - return 0; } From 86959c7d009f60c57e1fd97a8f65fc40f6d5881c Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Tue, 15 Sep 2026 17:09:33 +0300 Subject: [PATCH 069/153] RDMA/mlx5: Fix signed integer overflow in EQE qp_srq type shift eqe->data.qp_srq.type is a u8. Shifting it left by MLX5_USER_INDEX_LEN (24) without a cast promotes it to signed int, causing undefined behaviour for values >= 128. All currently used type values are well below 128, so this does not manifest with current hardware. Cast to u32 before the shift to make the code well-defined across the full range. Signed-off-by: Yishai Hadas Link: https://patch.msgid.link/20260915140933.40580-16-yishaih@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/qpc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c index 8b5bd33a7f0f..ce2ef8828c09 100644 --- a/drivers/infiniband/hw/mlx5/qpc.c +++ b/drivers/infiniband/hw/mlx5/qpc.c @@ -128,7 +128,7 @@ static int rsc_event_notifier(struct notifier_block *nb, case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; - rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN); + rsn |= ((u32)eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN); break; default: return NOTIFY_DONE; From 9728effe43d8fc4d58e12b697d0d6127355abdd5 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Sat, 19 Sep 2026 18:08:48 +0800 Subject: [PATCH 070/153] RDMA/mlx5: Restore mr->pi_mr after temporary stack-local assignment In handle_reg_mr_integrity, when mr->pi_mr is NULL (physical-address optimization path), the code points mr->pi_mr at a stack-local pa_pi_mr so set_pi_umr_wr can read it during this call. The pointer is never restored, so after the function returns mr->pi_mr dangles at a freed stack frame. A second IB_WR_REG_MR_INTEGRITY posted on the same MR without remapping reads the stale pointer and dereferences it. Save the original pi_mr (captured at function entry) and restore it on the out path. Fixes: 2563e2f30acb ("RDMA/mlx5: Use PA mapping for PI handover") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260919100848.2442-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/wr.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c index fca9e1d9d5e9..bb2c60380fe2 100644 --- a/drivers/infiniband/hw/mlx5/wr.c +++ b/drivers/infiniband/hw/mlx5/wr.c @@ -927,6 +927,7 @@ static noinline_for_stack int handle_reg_mr_integrity(struct mlx5_ib_dev *dev, qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; out: + mr->pi_mr = pi_mr; return err; } From d30e9b1bc47bd9be4b2fc97b0e4074bc02c4203a Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Mon, 21 Sep 2026 11:11:00 +0800 Subject: [PATCH 071/153] IB/isert: Don't share drop_cmd_list across concurrent connection teardowns isert_put_unsol_pending_cmds() declares drop_cmd_list as a static LIST_HEAD, so the list is shared across all invocations of the function. It is called from isert_wait_conn(), the per-connection .iscsit_wait_conn transport callback, which runs independently for each connection during teardown. conn->cmd_lock only protects each connection's own conn_cmd_list, not the shared drop_cmd_list. If two connections tear down at the same time, both threads list_move_tail() commands into the same drop_cmd_list without any mutual exclusion, and then both iterate and list_del/put it concurrently. This corrupts the linked list and can double-free iscsit_cmd structures. Make drop_cmd_list a stack-local list, matching the pattern already used in isert_free_np(). Fixes: 3e03c4b01da3 ("iser-target: Put the reference on commands waiting for unsol data") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20260921031100.2199-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/isert/ib_isert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 5087ea983071..43fe5d7cc15f 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -2565,7 +2565,7 @@ static void isert_put_unsol_pending_cmds(struct iscsit_conn *conn) { struct iscsit_cmd *cmd, *tmp; - static LIST_HEAD(drop_cmd_list); + LIST_HEAD(drop_cmd_list); spin_lock_bh(&conn->cmd_lock); list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { From 8b55ae7c81f0388e8f161b78e68eb65911f4959b Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 23 Sep 2026 06:11:51 -0700 Subject: [PATCH 072/153] RDMA/mana_ib: Optimize shadow queue bookkeeping Pack the posted WQE size and send opcode into compact bitfields. Move the UD receive shadow fields into the common shadow_wqe_header. This simplifies the code and lets all QP and WQ types share one structure. Use release and acquire operations so posting and polling can safely run on different CPUs. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/20260923131155.4055875-2-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 35 ++++++++++++----------- drivers/infiniband/hw/mana/qp.c | 4 +-- drivers/infiniband/hw/mana/shadow_queue.h | 29 +++++++++---------- drivers/infiniband/hw/mana/wr.c | 15 +++++----- 4 files changed, 40 insertions(+), 43 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index de81a4b994e0..ef39f5d35c48 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -190,15 +190,15 @@ static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe { struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; - struct ud_sq_shadow_wqe *shadow_wqe; + struct shadow_wqe_header *shadow_wqe; shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq); if (!shadow_wqe) return; - shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error; + shadow_wqe->error_code = rdma_cqe->ud_send.vendor_error; - wq->tail += shadow_wqe->header.posted_wqe_size; + wq->tail += shadow_wqe->wqe_size_in_bu; shadow_queue_advance_next_to_complete(&qp->shadow_sq); } @@ -206,7 +206,7 @@ static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe { struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; - struct ud_rq_shadow_wqe *shadow_wqe; + struct shadow_wqe_header *shadow_wqe; shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq); if (!shadow_wqe) @@ -214,9 +214,9 @@ static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; - shadow_wqe->header.error_code = IB_WC_SUCCESS; + shadow_wqe->error_code = IB_WC_SUCCESS; - wq->tail += shadow_wqe->header.posted_wqe_size; + wq->tail += shadow_wqe->wqe_size_in_bu; shadow_queue_advance_next_to_complete(&qp->shadow_rq); } @@ -238,21 +238,20 @@ static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe) } static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc, - const struct shadow_wqe_header *shadow_wqe) + const struct shadow_wqe_header *shadow_wqe, + enum ib_wc_opcode opcode) { - const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe; - wc->wr_id = shadow_wqe->wr_id; wc->status = shadow_wqe->error_code; - wc->opcode = shadow_wqe->opcode; + wc->opcode = opcode; wc->vendor_err = shadow_wqe->error_code; wc->wc_flags = 0; wc->qp = &qp->ibqp; wc->pkey_index = 0; - if (shadow_wqe->opcode == IB_WC_RECV) { - wc->byte_len = ud_wqe->byte_len; - wc->src_qp = ud_wqe->src_qpn; + if (opcode == IB_WC_RECV) { + wc->byte_len = shadow_wqe->byte_len; + wc->src_qp = shadow_wqe->src_qpn; wc->wc_flags |= IB_WC_GRH; } } @@ -270,7 +269,8 @@ static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc if (wc_index >= nwc) goto out; - fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); + fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe, + shadow_wqe->send_opcode); shadow_queue_advance_consumer(&qp->shadow_sq); wc_index++; } @@ -283,7 +283,8 @@ static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc if (wc_index >= nwc) goto out; - fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); + fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe, + IB_WC_RECV); shadow_queue_advance_consumer(&qp->shadow_rq); wc_index++; } @@ -296,13 +297,13 @@ static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc static void mana_drain_gsi_sq(struct mana_ib_qp *qp) { struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); - struct ud_sq_shadow_wqe *shadow_wqe; + struct shadow_wqe_header *shadow_wqe; unsigned long flags; spin_lock_irqsave(&cq->cq_lock, flags); while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) != NULL) { - shadow_wqe->header.error_code = IB_WC_GENERAL_ERR; + shadow_wqe->error_code = IB_WC_GENERAL_ERR; shadow_queue_advance_next_to_complete(&qp->shadow_sq); } spin_unlock_irqrestore(&cq->cq_lock, flags); diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 88e792cb0f85..a95c736d496f 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -749,13 +749,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, doorbell = mdev->gdma_dev->doorbell; err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr, - sizeof(struct ud_rq_shadow_wqe)); + sizeof(struct shadow_wqe_header)); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err); goto destroy_queues; } err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr, - sizeof(struct ud_sq_shadow_wqe)); + sizeof(struct shadow_wqe_header)); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err); goto destroy_shadow_queues; diff --git a/drivers/infiniband/hw/mana/shadow_queue.h b/drivers/infiniband/hw/mana/shadow_queue.h index a4b3818f9c39..a127edf7a84c 100644 --- a/drivers/infiniband/hw/mana/shadow_queue.h +++ b/drivers/infiniband/hw/mana/shadow_queue.h @@ -7,22 +7,14 @@ #define _MANA_SHADOW_QUEUE_H_ struct shadow_wqe_header { - u16 opcode; - u16 error_code; - u32 posted_wqe_size; u64 wr_id; -}; - -struct ud_rq_shadow_wqe { - struct shadow_wqe_header header; + u64 wqe_size_in_bu : 8; + u64 send_opcode : 4; + u64 error_code : 16; u32 byte_len; u32 src_qpn; }; -struct ud_sq_shadow_wqe { - struct shadow_wqe_header header; -}; - struct shadow_queue { /* Unmasked producer index, Incremented on wqe posting */ u64 prod_idx; @@ -57,12 +49,14 @@ static inline void destroy_shadow_queue(struct shadow_queue *queue) static inline bool shadow_queue_full(struct shadow_queue *queue) { - return (queue->prod_idx - queue->cons_idx) >= queue->length; + /* Do not reuse an entry until the poller has finished reading it. */ + return (queue->prod_idx - smp_load_acquire(&queue->cons_idx)) >= queue->length; } static inline bool shadow_queue_empty(struct shadow_queue *queue) { - return queue->prod_idx == queue->cons_idx; + /* Pair with posting's release of the initialized shadow WQE. */ + return smp_load_acquire(&queue->prod_idx) == queue->cons_idx; } static inline void * @@ -91,7 +85,8 @@ shadow_queue_get_next_to_consume(const struct shadow_queue *queue) static inline void * shadow_queue_get_next_to_complete(struct shadow_queue *queue) { - if (queue->next_to_complete_idx == queue->prod_idx) + /* Observe initialized WQE fields published by the posting CPU. */ + if (queue->next_to_complete_idx == smp_load_acquire(&queue->prod_idx)) return NULL; return shadow_queue_get_element(queue, queue->next_to_complete_idx); @@ -99,12 +94,14 @@ shadow_queue_get_next_to_complete(struct shadow_queue *queue) static inline void shadow_queue_advance_producer(struct shadow_queue *queue) { - queue->prod_idx++; + /* Publish all WQE fields to CQ polling on another CPU. */ + smp_store_release(&queue->prod_idx, queue->prod_idx + 1); } static inline void shadow_queue_advance_consumer(struct shadow_queue *queue) { - queue->cons_idx++; + /* Finish WC generation and queue-tail updates before allowing reuse. */ + smp_store_release(&queue->cons_idx, queue->cons_idx + 1); } static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue) diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c index 36a1d506f08f..8cd2980d4934 100644 --- a/drivers/infiniband/hw/mana/wr.c +++ b/drivers/infiniband/hw/mana/wr.c @@ -14,7 +14,7 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr * struct gdma_posted_wqe_info wqe_info = {0}; struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM]; struct gdma_wqe_request wqe_req = {0}; - struct ud_rq_shadow_wqe *shadow_wqe; + struct shadow_wqe_header *shadow_wqe; int err, i; if (shadow_queue_full(&qp->shadow_rq)) @@ -37,9 +37,8 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr * shadow_wqe = shadow_queue_producer_entry(&qp->shadow_rq); memset(shadow_wqe, 0, sizeof(*shadow_wqe)); - shadow_wqe->header.opcode = IB_WC_RECV; - shadow_wqe->header.wr_id = wr->wr_id; - shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu; + shadow_wqe->wr_id = wr->wr_id; + shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; shadow_queue_advance_producer(&qp->shadow_rq); mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); @@ -82,7 +81,7 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr struct gdma_posted_wqe_info wqe_info = {0}; struct gdma_wqe_request wqe_req = {0}; struct rdma_send_oob send_oob = {0}; - struct ud_sq_shadow_wqe *shadow_wqe; + struct shadow_wqe_header *shadow_wqe; int err, i; if (!ndev) { @@ -132,9 +131,9 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr qp->ud_qp.sq_psn++; shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq); memset(shadow_wqe, 0, sizeof(*shadow_wqe)); - shadow_wqe->header.opcode = IB_WC_SEND; - shadow_wqe->header.wr_id = wr->wr.wr_id; - shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu; + shadow_wqe->send_opcode = IB_WC_SEND; + shadow_wqe->wr_id = wr->wr.wr_id; + shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; shadow_queue_advance_producer(&qp->shadow_sq); mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); From afcebf3cfa0e84ea7d4399832d4b5edb24756f09 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 23 Sep 2026 06:11:52 -0700 Subject: [PATCH 073/153] RDMA/mana_ib: Revise UD send posting and WQE definitions Batch multiple send requests under a single doorbell. Move the send PSN to the common QP structure so it can be shared by all QP types. Define new WQE formats for UD and RC QPs. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/20260923131155.4055875-3-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/mana_ib.h | 40 ++++++++++++++++- drivers/infiniband/hw/mana/wr.c | 64 ++++++++++++++++------------ 2 files changed, 75 insertions(+), 29 deletions(-) diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index 8d63bc167e42..ec4101664ccd 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -206,7 +206,6 @@ enum mana_ud_queue_type { struct mana_ib_ud_qp { struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX]; - u32 sq_psn; }; struct mana_ib_qp { @@ -222,6 +221,7 @@ struct mana_ib_qp { /* The port on the IB device, starting with 1 */ u32 port; + u32 sq_psn; struct list_head cq_send_list; struct list_head cq_recv_list; @@ -525,7 +525,16 @@ struct mana_rnic_set_qp_state_resp { enum WQE_OPCODE_TYPES { WQE_TYPE_UD_SEND = 0, + WQE_TYPE_RC_SEND = 2, + WQE_TYPE_RC_SEND_IMM = 3, + WQE_TYPE_RC_SEND_INV = 4, + WQE_TYPE_WRITE = 5, + WQE_TYPE_WRITE_IMM = 6, + WQE_TYPE_READ = 7, WQE_TYPE_UD_RECV = 8, + WQE_TYPE_RC_RECV = 9, + WQE_TYPE_REG_MR = 10, + WQE_TYPE_LOCAL_INV = 12, }; /* HW DATA */ struct rdma_send_oob { @@ -544,7 +553,36 @@ struct rdma_send_oob { u32 reserved1; u32 reserved2; } ud_send; + union { + u32 immediate; + u32 invalidate_key; + } rc_send; + struct { + u32 address_hi; + u32 address_low; + u32 rkey; + u32 dma_len; + } rdma; + struct { + u32 mkey; + } mm; }; + union { + u32 immediate_ext; + struct { + u16 rsn; + u16 reserved; + } read; + }; + u32 fsn : 24; + u32 reserved2 : 8; +}; /* HW DATA */ + +struct rdma_recv_oob { + u32 psn_start : 24; + u32 reserved1 : 8; + u32 msn : 24; + u32 reserved2 : 8; }; /* HW DATA */ struct mana_rdma_cqe { diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c index 8cd2980d4934..f25df2fad914 100644 --- a/drivers/infiniband/hw/mana/wr.c +++ b/drivers/infiniband/hw/mana/wr.c @@ -71,7 +71,7 @@ int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, return err; } -static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr) +static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr *wr) { struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); struct mana_ib_ah *ah = container_of(wr->ah, struct mana_ib_ah, ibah); @@ -84,11 +84,8 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr struct shadow_wqe_header *shadow_wqe; int err, i; - if (!ndev) { - ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n", - qp->port, qp->ibqp.qp_num); + if (!ndev) return -EINVAL; - } if (wr->wr.opcode != IB_WR_SEND) return -EINVAL; @@ -110,7 +107,7 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr wqe_req.num_sge = wr->wr.num_sge + 1; wqe_req.sgl = gdma_sgl; - wqe_req.inline_oob_size = sizeof(struct rdma_send_oob); + wqe_req.inline_oob_size = INLINE_OOB_LARGE_SIZE; wqe_req.inline_oob_data = &send_oob; wqe_req.flags = GDMA_WR_OOB_IN_SGL; wqe_req.client_data_unit = ib_mtu_enum_to_int(ib_mtu_int_to_enum(ndev->mtu)); @@ -119,7 +116,7 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr send_oob.fence = !!(wr->wr.send_flags & IB_SEND_FENCE); send_oob.signaled = !!(wr->wr.send_flags & IB_SEND_SIGNALED); send_oob.solicited = !!(wr->wr.send_flags & IB_SEND_SOLICITED); - send_oob.psn = qp->ud_qp.sq_psn; + send_oob.psn = qp->sq_psn; send_oob.ssn_or_rqpn = wr->remote_qpn; send_oob.ud_send.remote_qkey = qp->ibqp.qp_type == IB_QPT_GSI ? IB_QP1_QKEY : wr->remote_qkey; @@ -128,40 +125,51 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr if (err) return err; - qp->ud_qp.sq_psn++; + qp->sq_psn++; shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq); memset(shadow_wqe, 0, sizeof(*shadow_wqe)); - shadow_wqe->send_opcode = IB_WC_SEND; shadow_wqe->wr_id = wr->wr.wr_id; shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; shadow_queue_advance_producer(&qp->shadow_sq); - mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); return 0; } +static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); + struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; + bool ring_sq = false; + int err = 0; + + for (; wr; wr = wr->next) { + err = mana_ib_post_send_ud_one(qp, ud_wr(wr)); + if (unlikely(err)) { + *bad_wr = wr; + break; + } + ring_sq = true; + } + + if (ring_sq) + mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); + + return err; +} + int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { - int err = 0; struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); - for (; wr; wr = wr->next) { - switch (ibqp->qp_type) { - case IB_QPT_UD: - case IB_QPT_GSI: - err = mana_ib_post_send_ud(qp, ud_wr(wr)); - if (unlikely(err)) { - *bad_wr = wr; - return err; - } - break; - default: - ibdev_dbg(ibqp->device, "Posting send wr on qp type %u is not supported\n", - ibqp->qp_type); - return -EINVAL; - } + switch (ibqp->qp_type) { + case IB_QPT_UD: + case IB_QPT_GSI: + return mana_ib_post_send_ud(qp, wr, bad_wr); + default: + /* Unsupported QP type */ + *bad_wr = wr; + return -EINVAL; } - - return err; } From 387c51aa0ee0aeeac4571d11aca759fa74dd3ed2 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 23 Sep 2026 06:11:53 -0700 Subject: [PATCH 074/153] RDMA/mana_ib: Revise UD receive posting with GDMA_WR_IB_SGL Decouple RQ WQE posting from the QP structure so it can be reused by other QP types. Batch multiple receive WQEs under a single doorbell. Introduce GDMA_WR_IB_SGL to pass SGEs directly to GDMA instead of copying them into a small stack array. Support receive WQEs with num_sge set to zero. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/20260923131155.4055875-4-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/wr.c | 76 +++++++++++-------- .../net/ethernet/microsoft/mana/gdma_main.c | 23 +++++- include/net/mana/gdma.h | 7 +- 3 files changed, 73 insertions(+), 33 deletions(-) diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c index f25df2fad914..eb671b45b297 100644 --- a/drivers/infiniband/hw/mana/wr.c +++ b/drivers/infiniband/hw/mana/wr.c @@ -7,29 +7,30 @@ #define MAX_WR_SGL_NUM (2) -static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr) +static int mana_ib_post_rq(struct mana_ib_qp *qp, const struct ib_recv_wr *wr) { - struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); - struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; + struct ib_sge mana_ib_dummy_sge = {.addr = 1, .length = 0, .lkey = 0}; + struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp); struct gdma_posted_wqe_info wqe_info = {0}; - struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM]; + struct gdma_queue *queue = ib_rq->kmem; struct gdma_wqe_request wqe_req = {0}; struct shadow_wqe_header *shadow_wqe; - int err, i; + int err; if (shadow_queue_full(&qp->shadow_rq)) return -EINVAL; - if (wr->num_sge > MAX_WR_SGL_NUM) + if (wr->num_sge > MAX_RX_WQE_SGL_ENTRIES) return -EINVAL; - for (i = 0; i < wr->num_sge; ++i) { - gdma_sgl[i].address = wr->sg_list[i].addr; - gdma_sgl[i].mem_key = wr->sg_list[i].lkey; - gdma_sgl[i].size = wr->sg_list[i].length; - } wqe_req.num_sge = wr->num_sge; - wqe_req.sgl = gdma_sgl; + wqe_req.ib_sgl = wr->sg_list; + wqe_req.flags = GDMA_WR_IB_SGL; + + if (wr->num_sge == 0) { + wqe_req.ib_sgl = &mana_ib_dummy_sge; + wqe_req.num_sge = 1; + } err = mana_gd_post_work_request(queue, &wqe_req, &wqe_info); if (err) @@ -41,34 +42,47 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr * shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; shadow_queue_advance_producer(&qp->shadow_rq); - mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); return 0; } +static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); + struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp); + struct gdma_queue *rq = ib_rq->kmem; + bool ring_rq = false; + int err = 0; + + for (; wr; wr = wr->next) { + err = mana_ib_post_rq(qp, wr); + if (unlikely(err)) { + *bad_wr = wr; + break; + } + ring_rq = true; + } + + if (ring_rq) + mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), rq); + + return err; +} + int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); - int err = 0; - for (; wr; wr = wr->next) { - switch (ibqp->qp_type) { - case IB_QPT_UD: - case IB_QPT_GSI: - err = mana_ib_post_recv_ud(qp, wr); - if (unlikely(err)) { - *bad_wr = wr; - return err; - } - break; - default: - ibdev_dbg(ibqp->device, "Posting recv wr on qp type %u is not supported\n", - ibqp->qp_type); - return -EINVAL; - } + switch (ibqp->qp_type) { + case IB_QPT_UD: + case IB_QPT_GSI: + return mana_ib_post_recv_ud(qp, wr, bad_wr); + default: + /* Unsupported QP type */ + *bad_wr = wr; + return -EINVAL; } - - return err; } static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr *wr) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index f92b2d0bf926..542491a6c86e 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -1738,6 +1738,24 @@ static void mana_gd_write_sgl(struct gdma_queue *wq, u32 sgl_offset, memcpy(mana_gd_ring_ptr(wq, sgl_offset), address, sgl_size); } +static void mana_gd_write_ib_sgl(struct gdma_queue *wq, u32 sgl_offset, + const struct gdma_wqe_request *wqe_req) +{ + const struct ib_sge *sge = wqe_req->ib_sgl; + struct gdma_sge *gdma_sgl; + u32 i; + + for (i = 0; i < wqe_req->num_sge; ++i, ++sge) { + gdma_sgl = mana_gd_ring_ptr(wq, sgl_offset); + gdma_sgl->address = sge->addr; + gdma_sgl->size = sge->length; + gdma_sgl->mem_key = sge->lkey; + sgl_offset += sizeof(*gdma_sgl); + if (sgl_offset == wq->queue_size) + sgl_offset = 0; + } +} + int mana_gd_post_work_request(struct gdma_queue *wq, const struct gdma_wqe_request *wqe_req, struct gdma_posted_wqe_info *wqe_info) @@ -1792,7 +1810,10 @@ int mana_gd_post_work_request(struct gdma_queue *wq, if (sgl_offset >= wq->queue_size) sgl_offset -= wq->queue_size; - mana_gd_write_sgl(wq, sgl_offset, wqe_req); + if (wqe_req->flags & GDMA_WR_IB_SGL) + mana_gd_write_ib_sgl(wq, sgl_offset, wqe_req); + else + mana_gd_write_sgl(wq, sgl_offset, wqe_req); wq->head += wqe_size / GDMA_WQE_BU_SIZE; diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 06af65d50eb7..8ccbe2d5738e 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -6,6 +6,7 @@ #include #include +#include #include "shm_channel.h" @@ -54,6 +55,7 @@ enum gdma_work_request_flags { GDMA_WR_NONE = 0, GDMA_WR_OOB_IN_SGL = BIT(0), GDMA_WR_PAD_BY_SGE0 = BIT(1), + GDMA_WR_IB_SGL = BIT(2), }; enum gdma_eqe_type { @@ -212,7 +214,10 @@ struct gdma_sge { }; /* HW DATA */ struct gdma_wqe_request { - struct gdma_sge *sgl; + union { + struct gdma_sge *sgl; + struct ib_sge *ib_sgl; + }; u32 num_sge; u32 inline_oob_size; From 9d3fccc58efe4a868474d6356db3fec80159dad7 Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 23 Sep 2026 06:11:54 -0700 Subject: [PATCH 075/153] RDMA/mana_ib: Make kernel CQ arming robust Track CQ polling credits and ring the doorbell promptly to advance the hardware CQ tail pointer. The distance between the new and previously armed tails must remain below four times the CQ length. Expose CQ arming to mana_ib through mana_gd_wq_ring_doorbell_ext(). RC QPs will also use this helper to ring special doorbell offsets. MANA hardware ignores duplicate doorbells. Calculate the arming offset from the previously armed offset and poll_credit to avoid duplicates. Do not arm the CQ if no CQE was produced, because the CQ remains armed. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/20260923131155.4055875-5-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 55 ++++++++++++++++++- drivers/infiniband/hw/mana/mana_ib.h | 1 + .../net/ethernet/microsoft/mana/gdma_main.c | 20 +++++-- include/net/mana/gdma.h | 2 + 4 files changed, 72 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index ef39f5d35c48..804030d177ea 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -64,6 +64,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, doorbell = mdev->gdma_dev->doorbell; } + ibcq->cqe = cq->cqe; + cq->poll_credit = (cq->cqe << (GDMA_CQE_OWNER_BITS - 1)) - 1; + if (is_rnic_cq) { err = mana_ib_gd_create_cq(mdev, cq, doorbell); if (err) { @@ -174,15 +177,62 @@ void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) gc->cq_table[cq->queue.id] = NULL; } +static inline bool gdma_cq_idx_produced(struct gdma_queue *gdma_cq, uint32_t idx) +{ + struct gdma_mem_info *gmi = &gdma_cq->mem_info; + u32 num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE; + u32 expected_bits = (idx / num_cqe) & GDMA_CQE_OWNER_MASK; + u32 offset = (idx % num_cqe) * GDMA_CQE_SIZE; + struct gdma_cqe *cqe; + + if (gmi->nr_pages) + cqe = gmi->pages_va[offset / PAGE_SIZE] + + (offset & (PAGE_SIZE - 1)); + else + cqe = gdma_cq->queue_mem_ptr + offset; + + return cqe->cqe_info.owner_bits == expected_bits; +} + +static inline void mana_ib_cq_doorbell(struct mana_ib_cq *cq, uint8_t arm) +{ + struct mana_ib_dev *mdev = container_of(cq->ibcq.device, struct mana_ib_dev, ib_dev); + struct gdma_queue *gdma_cq = cq->queue.kmem; + u32 num_cqe, max_credit, idx; + + num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE; + max_credit = num_cqe << (GDMA_CQE_OWNER_BITS - 1); + idx = gdma_cq->head; + + if (cq->poll_credit >= max_credit) { + if (gdma_cq_idx_produced(gdma_cq, idx + cq->poll_credit - max_credit)) + cq->poll_credit++; + else + return; + } else { + /* Set index of already polled CQE for unarm */ + cq->poll_credit = max_credit - (arm ? 0 : 1); + } + + idx += (cq->poll_credit - max_credit); + idx %= (num_cqe << GDMA_CQE_OWNER_BITS); + + mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), gdma_cq, idx, arm, 0); +} + int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); struct gdma_queue *gdma_cq = cq->queue.kmem; + unsigned long irq_flags; if (!gdma_cq) return -EINVAL; - mana_gd_ring_cq(gdma_cq, SET_ARM_BIT); + spin_lock_irqsave(&cq->cq_lock, irq_flags); + mana_ib_cq_doorbell(cq, SET_ARM_BIT); + spin_unlock_irqrestore(&cq->cq_lock, irq_flags); + return 0; } @@ -343,6 +393,9 @@ int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); if (comp_read < 1) break; + cq->poll_credit--; + if (!cq->poll_credit) + mana_ib_cq_doorbell(cq, 0); mana_handle_cqe(mdev, &gdma_cqe); } diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index ec4101664ccd..aa8c81d72219 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -170,6 +170,7 @@ struct mana_ib_cq { struct list_head list_recv_qp; int cqe; u32 comp_vector; + u32 poll_credit; mana_handle_t cq_handle; }; diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 542491a6c86e..cb51fcf579c8 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -559,11 +559,13 @@ static int mana_gd_disable_queue(struct gdma_queue *queue) static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index, enum gdma_queue_type q_type, u32 qid, - u32 tail_ptr, u8 num_req) + u32 tail_ptr, u8 num_req, u8 client_offset) { void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index; union gdma_doorbell_entry e = {}; + addr += client_offset; + switch (q_type) { case GDMA_EQ: e.eq.id = qid; @@ -623,10 +625,18 @@ void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue) * wqe_cnt for Receive Queues. This value is not used in Send Queues. */ mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, - queue->id, queue->head * GDMA_WQE_BU_SIZE, 0); + queue->id, queue->head * GDMA_WQE_BU_SIZE, 0, 0); } EXPORT_SYMBOL_NS(mana_gd_wq_ring_doorbell, "NET_MANA"); +void mana_gd_wq_ring_doorbell_ext(struct gdma_context *gc, struct gdma_queue *queue, + u32 tail_ptr, u8 wqe_cnt, u8 client_offset) +{ + mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, + queue->id, tail_ptr, wqe_cnt, client_offset); +} +EXPORT_SYMBOL_NS(mana_gd_wq_ring_doorbell_ext, "NET_MANA"); + void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) { struct gdma_context *gc = cq->gdma_dev->gdma_context; @@ -636,7 +646,7 @@ void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS); mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id, - head, arm_bit); + head, arm_bit, 0); } EXPORT_SYMBOL_NS(mana_gd_ring_cq, "NET_MANA"); @@ -653,7 +663,7 @@ void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld, dim_val |= MANA_INTR_MODR_USEC_VLD; mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, GDMA_DIM, cq->id, - dim_val, mod_comps_vld); + dim_val, mod_comps_vld, 0); } EXPORT_SYMBOL_NS(mana_gd_ring_dim, "NET_MANA"); @@ -1023,7 +1033,7 @@ static void mana_gd_process_eq_events(void *arg) head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS); mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id, - head, SET_ARM_BIT); + head, SET_ARM_BIT, 0); } static int mana_gd_register_irq(struct gdma_queue *queue, diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 8ccbe2d5738e..02632acd7848 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -1068,6 +1068,8 @@ void mana_gd_free_res_map(struct gdma_resource *r); void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue); +void mana_gd_wq_ring_doorbell_ext(struct gdma_context *gc, struct gdma_queue *queue, + u32 tail_ptr, u8 wqe_cnt, u8 client_offset); int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, struct gdma_mem_info *gmi, bool allow_scatter); From 1d99b7cea29c315e12cc4687f63c1d626437562e Mon Sep 17 00:00:00 2001 From: Konstantin Taranov Date: Wed, 23 Sep 2026 06:11:55 -0700 Subject: [PATCH 076/153] RDMA/mana_ib: Poll UD completions and flush software error QPs Build WCs directly from CQEs and their corresponding published shadow entries. This avoids iterating over all QPs that share a CQ. Track only QPs in the error state instead of every QP associated with the CQ. Keep the explicit GSI send-drain path responsible for adding the QP to the send error list and invoking its CQ handler. To invoke the handler for WQEs posted after GSI draining, check the QP state after posting WQEs. This is required because MANA hardware ignores doorbells for QPs in the error state. Introduce a WC-budgeted polling context that caches CQEs representing multiple WCs. Before consuming a shadow entry, match the CQE against the low 24 bits of the posted WQE offset to filter out stale CQEs. Implement WR signaling to skip WC generation for unsignaled WQEs. Signed-off-by: Konstantin Taranov Link: https://patch.msgid.link/20260923131155.4055875-6-kotaranov@linux.microsoft.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mana/cq.c | 320 ++++++++++++------ drivers/infiniband/hw/mana/mana_ib.h | 64 +++- drivers/infiniband/hw/mana/qp.c | 80 +++-- drivers/infiniband/hw/mana/shadow_queue.h | 38 +-- drivers/infiniband/hw/mana/wr.c | 19 +- .../net/ethernet/microsoft/mana/gdma_main.c | 8 +- include/net/mana/gdma.h | 2 + 7 files changed, 371 insertions(+), 160 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index 804030d177ea..843e2c64b32b 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -5,6 +5,58 @@ #include "mana_ib.h" +static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error) +{ + switch (vendor_error) { + case VENDOR_ERR_OK: + return IB_WC_SUCCESS; + case VENDOR_ERR_RX_PKT_LEN: + case VENDOR_ERR_RX_MSG_LEN_OVFL: + return IB_WC_LOC_LEN_ERR; + case VENDOR_ERR_TX_GDMA_CORRUPTED_WQE: + case VENDOR_ERR_TX_PCIE_WQE: + case VENDOR_ERR_TX_PCIE_MSG: + case VENDOR_ERR_RX_MALFORMED_WQE: + case VENDOR_ERR_TX_GDMA_INVALID_STATE: + case VENDOR_ERR_TX_MISBEHAVING_CLIENT: + case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE: + case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD: + case VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED: + case VENDOR_ERR_TX_RDMA_WQE_LEN_ERR: + case VENDOR_ERR_TX_RDMA_MTU_ERR: + return IB_WC_LOC_QP_OP_ERR; + case VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION: + case VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE: + case VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR: + case VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION: + case VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE: + case VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR: + case VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE: + case VENDOR_ERR_RX_ATB_SGE_MISSCONFIG: + return IB_WC_LOC_PROT_ERR; + case VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT: + case VENDOR_ERR_RX_GFID: + return IB_WC_LOC_ACCESS_ERR; + case VENDOR_ERR_RX_MISBEHAVING_CLIENT: + case VENDOR_ERR_RX_CLIENT_ID: + case VENDOR_ERR_RX_PCIE: + case VENDOR_ERR_RX_NO_AVAIL_WQE: + case VENDOR_ERR_RX_ATB_WQE_MISCONFIG: + case VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT: + case VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE: + case VENDOR_ERR_TX_RDMA_INVALID_STATE: + case VENDOR_ERR_TX_RDMA_INVALID_NPT: + case VENDOR_ERR_TX_RDMA_INVALID_SGID: + case VENDOR_ERR_TX_RDMA_VFID_MISMATCH: + return IB_WC_FATAL_ERR; + case VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE: + case VENDOR_ERR_SW_FLUSHED: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } +} + int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { @@ -89,8 +141,8 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } spin_lock_init(&cq->cq_lock); - INIT_LIST_HEAD(&cq->list_send_qp); - INIT_LIST_HEAD(&cq->list_recv_qp); + INIT_LIST_HEAD(&cq->send_err_qp_list); + INIT_LIST_HEAD(&cq->recv_err_qp_list); return 0; @@ -236,126 +288,172 @@ int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) return 0; } -static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) +struct mana_cq_poll { + struct ib_wc *wc; + int budget; + int produced; +}; + +static struct ib_wc *mana_fill_wc(struct mana_ib_qp *qp, + struct mana_cq_poll *poll, + const struct shadow_wqe_header *wqe, + enum ib_wc_opcode opcode, u32 vendor_error) { - struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; - struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; - struct shadow_wqe_header *shadow_wqe; + struct ib_wc *wc = &poll->wc[poll->produced++]; - shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq); - if (!shadow_wqe) - return; + memset(wc, 0, sizeof(*wc)); + wc->wr_id = wqe->wr_id; + wc->status = vendor_error_to_wc_error(vendor_error); + wc->opcode = opcode; + wc->vendor_err = vendor_error; + wc->qp = &qp->ibqp; - shadow_wqe->error_code = rdma_cqe->ud_send.vendor_error; - - wq->tail += shadow_wqe->wqe_size_in_bu; - shadow_queue_advance_next_to_complete(&qp->shadow_sq); + return wc; } -static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) +static void mana_complete_send(struct mana_ib_qp *qp, + struct mana_cq_poll *poll, u32 vendor_error) { - struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; - struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; - struct shadow_wqe_header *shadow_wqe; + struct shadow_queue *shadow = &qp->shadow_sq; + struct shadow_wqe_header *wqe = shadow_queue_get_next_to_consume(shadow); + struct gdma_queue *queue; - shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq); - if (!shadow_wqe) + if (!wqe) return; - shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; - shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; - shadow_wqe->error_code = IB_WC_SUCCESS; + if (vendor_error || !(wqe->flags & MANA_WQ_NO_SIGNAL_WC)) + mana_fill_wc(qp, poll, wqe, wqe->send_opcode, vendor_error); - wq->tail += shadow_wqe->wqe_size_in_bu; - shadow_queue_advance_next_to_complete(&qp->shadow_rq); + queue = mana_qp_get_sq(qp)->kmem; + queue->tail += wqe->wqe_size_in_bu; + shadow_queue_advance_consumer(shadow); } -static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe) +static void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct mana_rdma_cqe *rdma_cqe, + struct mana_cq_poll *poll) { + u32 offset = rdma_cqe->ud_send.tx_wqe_offset & MANA_WQE_OFFSET_MASK; + struct shadow_queue *shadow = &qp->shadow_sq; + struct shadow_wqe_header *wqe; + u64 idx = shadow->cons_idx; + u32 to_complete = 0; + u64 prod_idx; + + /* Pair with posting's release of the initialized shadow entries. */ + prod_idx = smp_load_acquire(&shadow->prod_idx); + /* Find the target before retiring any entries: the CQE may be stale. */ + for (; idx != prod_idx; idx++) { + wqe = shadow_queue_get_element(shadow, idx); + to_complete++; + if (wqe->wqe_offset_or_psn == offset) + break; + if (!(wqe->flags & MANA_WQ_NO_SIGNAL_WC)) + return; + } + if (idx == prod_idx) + return; + + for (; to_complete; to_complete--) + mana_complete_send(qp, poll, VENDOR_ERR_OK); +} + +static void handle_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe, + struct mana_cq_poll *poll) +{ + struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; + u32 offset = rdma_cqe->ud_recv.rx_wqe_offset / GDMA_WQE_BU_SIZE; + struct mana_ib_queue *rq = mana_qp_get_rq(qp); + struct gdma_queue *wq = rq->kmem; + struct shadow_wqe_header *wqe; + struct ib_wc *wc; + + wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq); + if (!wqe || wqe->wqe_offset_or_psn != (offset & MANA_WQE_OFFSET_MASK)) + return; + + wc = mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_OK); + switch (rdma_cqe->cqe_type) { + case CQE_TYPE_UD_SEND_IMM: + wc->ex.imm_data = cpu_to_be32(rdma_cqe->ud_recv.imm_data); + wc->wc_flags |= IB_WC_WITH_IMM; + fallthrough; + case CQE_TYPE_UD_SEND: + wc->byte_len = rdma_cqe->ud_recv.msg_len; + wc->src_qp = rdma_cqe->ud_recv.src_qpn; + wc->wc_flags |= IB_WC_GRH; + break; + default: + break; + } + + wq->tail += wqe->wqe_size_in_bu; + shadow_queue_advance_consumer(&qp->shadow_rq); +} + +static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev, + struct mana_cq_poll *poll) +{ + struct gdma_comp *cqe = &cq->pending_cqe; + struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq); if (!qp) + return true; + + switch (rdma_cqe->cqe_type) { + case CQE_TYPE_UD_SEND: + if (cqe->is_sq) { + handle_ud_sq_cqe(qp, rdma_cqe, poll); + break; + } + fallthrough; + case CQE_TYPE_UD_SEND_IMM: + handle_rq_cqe(qp, cqe, poll); + break; + default: + ibdev_warn_ratelimited(qp->ibqp.device, "Unexpected CQE type %u\n", + rdma_cqe->cqe_type); + break; + } + mana_put_qp_ref(qp); + return true; +} + +static void mana_flush_completions(struct mana_ib_cq *cq, struct mana_cq_poll *poll) +{ + struct shadow_wqe_header *wqe; + struct mana_ib_qp *qp; + + if (poll->produced >= poll->budget) return; - if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) { - if (cqe->is_sq) - handle_ud_sq_cqe(qp, cqe); - else - handle_ud_rq_cqe(qp, cqe); + list_for_each_entry(qp, &cq->send_err_qp_list, send_err_node) { + while (poll->produced < poll->budget && + shadow_queue_get_next_to_consume(&qp->shadow_sq)) + mana_complete_send(qp, poll, VENDOR_ERR_SW_FLUSHED); + if (poll->produced == poll->budget) + return; } - mana_put_qp_ref(qp); -} - -static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc, - const struct shadow_wqe_header *shadow_wqe, - enum ib_wc_opcode opcode) -{ - wc->wr_id = shadow_wqe->wr_id; - wc->status = shadow_wqe->error_code; - wc->opcode = opcode; - wc->vendor_err = shadow_wqe->error_code; - wc->wc_flags = 0; - wc->qp = &qp->ibqp; - wc->pkey_index = 0; - - if (opcode == IB_WC_RECV) { - wc->byte_len = shadow_wqe->byte_len; - wc->src_qp = shadow_wqe->src_qpn; - wc->wc_flags |= IB_WC_GRH; - } -} - -static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc) -{ - struct shadow_wqe_header *shadow_wqe; - struct mana_ib_qp *qp; - int wc_index = 0; - - /* process send shadow queue completions */ - list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { - while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) - != NULL) { - if (wc_index >= nwc) - goto out; - - fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe, - shadow_wqe->send_opcode); - shadow_queue_advance_consumer(&qp->shadow_sq); - wc_index++; - } - } - - /* process recv shadow queue completions */ - list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { - while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq)) - != NULL) { - if (wc_index >= nwc) - goto out; - - fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe, - IB_WC_RECV); + list_for_each_entry(qp, &cq->recv_err_qp_list, recv_err_node) { + while (poll->produced < poll->budget && + (wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))) { + mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_SW_FLUSHED); shadow_queue_advance_consumer(&qp->shadow_rq); - wc_index++; } + if (poll->produced == poll->budget) + return; } - -out: - return wc_index; } static void mana_drain_gsi_sq(struct mana_ib_qp *qp) { struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); - struct shadow_wqe_header *shadow_wqe; unsigned long flags; spin_lock_irqsave(&cq->cq_lock, flags); - while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) - != NULL) { - shadow_wqe->error_code = IB_WC_GENERAL_ERR; - shadow_queue_advance_next_to_complete(&qp->shadow_sq); - } + if (list_empty(&qp->send_err_node)) + list_add_tail(&qp->send_err_node, &cq->send_err_qp_list); spin_unlock_irqrestore(&cq->cq_lock, flags); if (cq->ibcq.comp_handler) @@ -382,25 +480,39 @@ int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev); + struct mana_cq_poll poll = { .wc = wc, .budget = num_entries, .produced = 0 }; struct gdma_queue *queue = cq->queue.kmem; - struct gdma_comp gdma_cqe; unsigned long flags; - int num_polled = 0; - int comp_read, i; + bool consumed; + int comp_read; + + if (!queue) + return -EINVAL; spin_lock_irqsave(&cq->cq_lock, flags); - for (i = 0; i < num_entries; i++) { - comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); - if (comp_read < 1) - break; - cq->poll_credit--; - if (!cq->poll_credit) - mana_ib_cq_doorbell(cq, 0); - mana_handle_cqe(mdev, &gdma_cqe); + while (poll.produced < poll.budget) { + if (!cq->has_pending_cqe) { + comp_read = mana_gd_poll_cq(queue, &cq->pending_cqe, 1); + if (comp_read < 0) { + if (!poll.produced) + poll.produced = comp_read; + goto out; + } + if (!comp_read) + break; + + cq->poll_credit--; + if (!cq->poll_credit) + mana_ib_cq_doorbell(cq, 0); + } + + consumed = mana_handle_cqe(cq, mdev, &poll); + cq->has_pending_cqe = !consumed; } - num_polled = mana_process_completions(cq, num_entries, wc); + mana_flush_completions(cq, &poll); +out: spin_unlock_irqrestore(&cq->cq_lock, flags); - return num_polled; + return poll.produced; } diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index aa8c81d72219..9173ca64d2e1 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -166,8 +166,10 @@ struct mana_ib_cq { struct mana_ib_queue queue; /* protects CQ polling */ spinlock_t cq_lock; - struct list_head list_send_qp; - struct list_head list_recv_qp; + struct list_head send_err_qp_list; + struct list_head recv_err_qp_list; + struct gdma_comp pending_cqe; + bool has_pending_cqe; int cqe; u32 comp_vector; u32 poll_credit; @@ -223,9 +225,13 @@ struct mana_ib_qp { /* The port on the IB device, starting with 1 */ u32 port; u32 sq_psn; + bool sq_sig_all; - struct list_head cq_send_list; - struct list_head cq_recv_list; + /* Serializes QP modification and error-list transitions. */ + struct mutex modify_lock; + + struct list_head send_err_node; + struct list_head recv_err_node; struct shadow_queue shadow_rq; struct shadow_queue shadow_sq; @@ -586,6 +592,53 @@ struct rdma_recv_oob { u32 reserved2 : 8; }; /* HW DATA */ +enum mana_ib_error_code { + VENDOR_ERR_OK = 0x0, + VENDOR_ERR_RX_PKT_LEN = 0x05, + VENDOR_ERR_RX_MSG_LEN_OVFL = 0x102, + VENDOR_ERR_RX_MISBEHAVING_CLIENT = 0x108, + VENDOR_ERR_RX_MALFORMED_WQE = 0x109, + VENDOR_ERR_RX_CLIENT_ID = 0x10a, + VENDOR_ERR_RX_GFID = 0x10b, + VENDOR_ERR_RX_PCIE = 0x10c, + VENDOR_ERR_RX_NO_AVAIL_WQE = 0x111, + VENDOR_ERR_RX_ATB_SGE_MISSCONFIG = 0x143, + VENDOR_ERR_RX_ATB_WQE_MISCONFIG = 0x145, + VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT = 0x183, + VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT = 0x185, + VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE = 0x1c3, + VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE = 0x1c5, + VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE = 0x1c7, + VENDOR_ERR_TX_GDMA_CORRUPTED_WQE = 0x201, + VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION = 0x202, + VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE = 0x203, + VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR = 0x204, + VENDOR_ERR_TX_PCIE_WQE = 0x205, + VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION = 0x206, + VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE = 0x207, + VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR = 0x208, + VENDOR_ERR_TX_PCIE_MSG = 0x209, + VENDOR_ERR_TX_GDMA_INVALID_STATE = 0x20a, + VENDOR_ERR_TX_MISBEHAVING_CLIENT = 0x20b, + VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE = 0x210, + VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD = 0x211, + VENDOR_ERR_TX_RDMA_INVALID_STATE = 0x212, + VENDOR_ERR_TX_RDMA_INVALID_NPT = 0x213, + VENDOR_ERR_TX_RDMA_INVALID_SGID = 0x214, + VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED = 0x215, + VENDOR_ERR_TX_RDMA_WQE_LEN_ERR = 0x216, + VENDOR_ERR_TX_RDMA_MTU_ERR = 0x217, + VENDOR_ERR_TX_RDMA_VFID_MISMATCH = 0x218, + VENDOR_ERR_HW_MAX = 0x3ff, + /* SW vendor errors */ + VENDOR_ERR_SW_FLUSHED = 0xfff, +}; + +enum mana_ib_cqe_type { + CQE_TYPE_UD_SEND = 1, + CQE_TYPE_UD_SEND_IMM = 2, +}; /* HW DATA */ + struct mana_rdma_cqe { union { struct { @@ -594,8 +647,7 @@ struct mana_rdma_cqe { }; struct { u32 cqe_type : 8; - u32 vendor_error : 9; - u32 reserved1 : 15; + u32 reserved1 : 24; u32 sge_offset : 5; u32 tx_wqe_offset : 27; } ud_send; diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index a95c736d496f..15c5d6cefd62 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -695,33 +695,23 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, return err; } -static void mana_add_qp_to_cqs(struct mana_ib_qp *qp) +static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp, bool reset) { struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); unsigned long flags; spin_lock_irqsave(&send_cq->cq_lock, flags); - list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp); + list_del_init(&qp->send_err_node); + /* Keep shadow reset serialized with hardware polling and SW flushing. */ + if (reset) + reset_shadow_queue(&qp->shadow_sq); spin_unlock_irqrestore(&send_cq->cq_lock, flags); spin_lock_irqsave(&recv_cq->cq_lock, flags); - list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp); - spin_unlock_irqrestore(&recv_cq->cq_lock, flags); -} - -static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp) -{ - struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); - struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); - unsigned long flags; - - spin_lock_irqsave(&send_cq->cq_lock, flags); - list_del(&qp->cq_send_list); - spin_unlock_irqrestore(&send_cq->cq_lock, flags); - - spin_lock_irqsave(&recv_cq->cq_lock, flags); - list_del(&qp->cq_recv_list); + list_del_init(&qp->recv_err_node); + if (reset) + reset_shadow_queue(&qp->shadow_rq); spin_unlock_irqrestore(&recv_cq->cq_lock, flags); } @@ -777,8 +767,6 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, if (err) goto destroy_qp; - mana_add_qp_to_cqs(qp); - return 0; destroy_qp: @@ -795,6 +783,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, struct ib_udata *udata) { + struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); + + qp->sq_sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR; + mutex_init(&qp->modify_lock); + INIT_LIST_HEAD(&qp->send_err_node); + INIT_LIST_HEAD(&qp->recv_err_node); + switch (attr->qp_type) { case IB_QPT_RAW_PACKET: /* When rwq_ind_tbl is used, it's for creating WQs for RSS */ @@ -892,19 +887,56 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, return 0; } +static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); + + if (udata) + return; + + if (attr_mask & IB_QP_STATE) { + switch (attr->qp_state) { + case IB_QPS_RESET: + mana_remove_qp_from_cqs(qp, true); + break; + default: + break; + } + } + + if (attr_mask & IB_QP_SQ_PSN) + qp->sq_psn = attr->sq_psn; +} + int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { + struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); + int ret; + + mutex_lock(&qp->modify_lock); + switch (ibqp->qp_type) { case IB_QPT_RC: case IB_QPT_UC: case IB_QPT_UD: case IB_QPT_GSI: - return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata); + ret = mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata); + if (ret) + goto out_unlock; + break; default: - ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; + goto out_unlock; } + + mana_ib_modify_qp_state(ibqp, attr, attr_mask, udata); + +out_unlock: + mutex_unlock(&qp->modify_lock); + + return ret; } static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, @@ -1041,8 +1073,8 @@ static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata) if (err) return err; - mana_remove_qp_from_cqs(qp); mana_table_remove_qp(mdev, qp); + mana_remove_qp_from_cqs(qp, false); destroy_shadow_queue(&qp->shadow_rq); destroy_shadow_queue(&qp->shadow_sq); diff --git a/drivers/infiniband/hw/mana/shadow_queue.h b/drivers/infiniband/hw/mana/shadow_queue.h index a127edf7a84c..b5d6bb28bd45 100644 --- a/drivers/infiniband/hw/mana/shadow_queue.h +++ b/drivers/infiniband/hw/mana/shadow_queue.h @@ -6,13 +6,19 @@ #ifndef _MANA_SHADOW_QUEUE_H_ #define _MANA_SHADOW_QUEUE_H_ +#include + +#define MANA_WQ_FENCE_WC BIT(0) +#define MANA_WQ_NO_SIGNAL_WC BIT(1) +#define MANA_WQE_OFFSET_MASK GENMASK(23, 0) + struct shadow_wqe_header { u64 wr_id; + u64 wqe_offset_or_psn : 24; u64 wqe_size_in_bu : 8; + u64 fsn : 24; u64 send_opcode : 4; - u64 error_code : 16; - u32 byte_len; - u32 src_qpn; + u64 flags : 2; }; struct shadow_queue { @@ -20,8 +26,6 @@ struct shadow_queue { u64 prod_idx; /* Unmasked consumer index, Incremented on cq polling */ u64 cons_idx; - /* Unmasked index of next-to-complete (from HW) shadow WQE */ - u64 next_to_complete_idx; /* queue size in wqes */ u32 length; /* distance between elements in bytes */ @@ -42,6 +46,12 @@ static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t lengt return 0; } +static inline void reset_shadow_queue(struct shadow_queue *queue) +{ + queue->prod_idx = 0; + queue->cons_idx = 0; +} + static inline void destroy_shadow_queue(struct shadow_queue *queue) { kvfree(queue->buffer); @@ -76,22 +86,13 @@ shadow_queue_producer_entry(struct shadow_queue *queue) static inline void * shadow_queue_get_next_to_consume(const struct shadow_queue *queue) { - if (queue->cons_idx == queue->next_to_complete_idx) + /* The producer publishes the WQE before advancing prod_idx. */ + if (queue->cons_idx == smp_load_acquire(&queue->prod_idx)) return NULL; return shadow_queue_get_element(queue, queue->cons_idx); } -static inline void * -shadow_queue_get_next_to_complete(struct shadow_queue *queue) -{ - /* Observe initialized WQE fields published by the posting CPU. */ - if (queue->next_to_complete_idx == smp_load_acquire(&queue->prod_idx)) - return NULL; - - return shadow_queue_get_element(queue, queue->next_to_complete_idx); -} - static inline void shadow_queue_advance_producer(struct shadow_queue *queue) { /* Publish all WQE fields to CQ polling on another CPU. */ @@ -104,9 +105,4 @@ static inline void shadow_queue_advance_consumer(struct shadow_queue *queue) smp_store_release(&queue->cons_idx, queue->cons_idx + 1); } -static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue) -{ - queue->next_to_complete_idx++; -} - #endif diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c index eb671b45b297..250d673f58f2 100644 --- a/drivers/infiniband/hw/mana/wr.c +++ b/drivers/infiniband/hw/mana/wr.c @@ -40,6 +40,7 @@ static int mana_ib_post_rq(struct mana_ib_qp *qp, const struct ib_recv_wr *wr) memset(shadow_wqe, 0, sizeof(*shadow_wqe)); shadow_wqe->wr_id = wr->wr_id; shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; + shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset; shadow_queue_advance_producer(&qp->shadow_rq); return 0; @@ -96,6 +97,7 @@ static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr struct gdma_wqe_request wqe_req = {0}; struct rdma_send_oob send_oob = {0}; struct shadow_wqe_header *shadow_wqe; + bool signaled = qp->sq_sig_all || (wr->wr.send_flags & IB_SEND_SIGNALED); int err, i; if (!ndev) @@ -128,7 +130,7 @@ static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr send_oob.wqe_type = WQE_TYPE_UD_SEND; send_oob.fence = !!(wr->wr.send_flags & IB_SEND_FENCE); - send_oob.signaled = !!(wr->wr.send_flags & IB_SEND_SIGNALED); + send_oob.signaled = signaled; send_oob.solicited = !!(wr->wr.send_flags & IB_SEND_SOLICITED); send_oob.psn = qp->sq_psn; send_oob.ssn_or_rqpn = wr->remote_qpn; @@ -143,7 +145,9 @@ static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq); memset(shadow_wqe, 0, sizeof(*shadow_wqe)); shadow_wqe->wr_id = wr->wr.wr_id; + shadow_wqe->flags = signaled ? 0 : MANA_WQ_NO_SIGNAL_WC; shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; + shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset; shadow_queue_advance_producer(&qp->shadow_sq); return 0; @@ -153,8 +157,11 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr * const struct ib_send_wr **bad_wr) { struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); + struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; + unsigned long flags; bool ring_sq = false; + bool flush_send; int err = 0; for (; wr; wr = wr->next) { @@ -166,9 +173,17 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr * ring_sq = true; } - if (ring_sq) + if (ring_sq) { mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); + spin_lock_irqsave(&cq->cq_lock, flags); + flush_send = !list_empty(&qp->send_err_node); + spin_unlock_irqrestore(&cq->cq_lock, flags); + + if (flush_send && cq->ibcq.comp_handler) + cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); + } + return err; } diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index cb51fcf579c8..05ba7055405d 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -1807,10 +1807,12 @@ int mana_gd_post_work_request(struct gdma_queue *wq, if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq)) return -ENOSPC; - if (wqe_info) - wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; - head = wq->head; + if (wqe_info) { + wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; + wqe_info->wqe_offset = head; + } + wqe_offset = (head * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); wqe_ptr = mana_gd_get_wqe_ptr(wq, head); oob_len = mana_gd_write_client_oob(wqe_req, wq->type, client_oob_size, diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 02632acd7848..2a28fcaa637d 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -630,6 +630,8 @@ struct gdma_eqe { struct gdma_posted_wqe_info { u32 wqe_size_in_bu; + /* Unmasked WQE start offset in GDMA basic units. */ + u32 wqe_offset; }; /* GDMA_GENERATE_TEST_EQE */ From 3483cbbd7487163345b4b4d5eb8cad8540d1c3ba Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 21 Sep 2026 23:12:23 -0700 Subject: [PATCH 077/153] RDMA/bnxt_re: Fix integer overflow in send payload size computation bnxt_re_build_sgl() summed SGE lengths into a signed int, which could overflow. Make it return u32, and have bnxt_re_copy_wr_payload() report the size via a u32 out-parameter with the return value carrying only the error status, instead of overloading a signed int with both. Fixes: 1ac5a4047975 ("RDMA/bnxt_re: Add bnxt_re RoCE driver") Signed-off-by: Selvin Xavier Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 44 +++++++++++++----------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 7f9aa3620abb..34a925c31385 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -165,10 +165,11 @@ static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev, qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO; } -static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, +static u32 bnxt_re_build_sgl(struct ib_sge *ib_sg_list, struct bnxt_qplib_sge *sg_list, int num) { - int i, total = 0; + u32 total = 0; + int i; for (i = 0; i < num; i++) { sg_list[i].addr = ib_sg_list[i].addr; @@ -3185,17 +3186,22 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, const struct ib_send_wr *wr, - struct bnxt_qplib_swqe *wqe) + struct bnxt_qplib_swqe *wqe, + u32 *payload_sz) { - int payload_sz = 0; + int rc; - if (wr->send_flags & IB_SEND_INLINE) - payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); - else - payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, - wqe->num_sge); + if (wr->send_flags & IB_SEND_INLINE) { + rc = bnxt_re_copy_inline_data(rdev, wr, wqe); + if (rc < 0) + return rc; + *payload_sz = rc; + } else { + *payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, + wqe->num_sge); + } - return payload_sz; + return 0; } static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) @@ -3218,7 +3224,8 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp, const struct ib_send_wr *wr) { - int rc = 0, payload_sz = 0; + int rc = 0; + u32 payload_sz = 0; unsigned long flags; spin_lock_irqsave(&qp->sq_lock, flags); @@ -3234,11 +3241,9 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, goto bad; } - payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); - if (payload_sz < 0) { - rc = -EINVAL; + rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz); + if (rc) goto bad; - } wqe.wr_id = wr->wr_id; wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; @@ -3279,7 +3284,8 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, { struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); struct bnxt_qplib_swqe wqe; - int rc = 0, payload_sz = 0; + int rc = 0; + u32 payload_sz = 0; unsigned long flags; spin_lock_irqsave(&qp->sq_lock, flags); @@ -3296,11 +3302,9 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, goto bad; } - payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); - if (payload_sz < 0) { - rc = -EINVAL; + rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz); + if (rc) goto bad; - } wqe.wr_id = wr->wr_id; switch (wr->opcode) { From 81551ece0864c346080d241719d458c93f0e7fc9 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 21 Sep 2026 23:12:24 -0700 Subject: [PATCH 078/153] RDMA/bnxt_re: Detect wrong sge_len passed for inline Avoid handling wrong sge_len by adding extra check to see if the passed length is more than the inline size supported. Fixes: 1ac5a4047975 ("RDMA/bnxt_re: Add bnxt_re RoCE driver") Reviewed-by: Kalesh AP Signed-off-by: Selvin Xavier Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 34a925c31385..9737c7e23dcc 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -3169,8 +3169,9 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, wr->sg_list[i].addr; sge_len = wr->sg_list[i].length; - if ((sge_len + wqe->inline_len) > - BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { + if (sge_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH || + ((sge_len + wqe->inline_len) > + BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH)) { ibdev_err(&rdev->ibdev, "Inline data size requested > supported value"); return -EINVAL; From 1894ed9a664d19c190ce1bd2146759efbae77e4b Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 21 Sep 2026 23:12:25 -0700 Subject: [PATCH 079/153] RDMA/bnxt_re: Validate SRQ max_sge at create time bnxt_re_create_srq() stored attr.max_sge into srq->qplib_srq.max_sge unvalidated, which defeats the num_sge check in bnxt_re_post_srq_recv() since that check compares against this same attacker-chosen value. Reject max_sge > dev_attr->max_srq_sges at create time, and clamp dev_attr->max_srq_sges to BNXT_STATIC_MAX_SGE since SRQ WQEs use the fixed-size SGE array. Fixes: 37cb11acf1f7 ("RDMA/bnxt_re: Add SRQ support for Broadcom adapters") Signed-off-by: Selvin Xavier Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 8 ++++++++ drivers/infiniband/hw/bnxt_re/qplib_sp.c | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 9737c7e23dcc..17bcaad51a89 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -2236,6 +2236,14 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, goto exit; } + if (srq_init_attr->attr.max_sge > dev_attr->max_srq_sges) { + ibdev_err(&rdev->ibdev, + "Create SRQ failed - max_sge %d exceeds supported %d", + srq_init_attr->attr.max_sge, dev_attr->max_srq_sges); + rc = -EINVAL; + goto exit; + } + if (srq_init_attr->srq_type != IB_SRQT_BASIC) { rc = -EOPNOTSUPP; goto exit; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c index ec9eb52a8ebf..9aaa2b5204b8 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c @@ -160,7 +160,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) attr->max_srq = le16_to_cpu(sb->max_srq); attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; - attr->max_srq_sges = sb->max_srq_sge; + attr->max_srq_sges = min_t(u32, sb->max_srq_sge, BNXT_STATIC_MAX_SGE); attr->max_pkey = 1; attr->max_inline_data = attr->max_qp_sges * sizeof(struct sq_sge); if (!bnxt_qplib_is_chip_gen_p7(rcfw->res->cctx)) From 4d8168a3b7c93385c11dc836df569d337089b218 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 21 Sep 2026 23:12:26 -0700 Subject: [PATCH 080/153] RDMA/bnxt_re: Initialize wqe.flags in bnxt_re_post_srq_recv() wqe is declared once above the WR loop and never zeroed, so wqe.flags holds indeterminate stack contents on the first WR and the previous WR's stale value thereafter. This propagates straight into the hardware-visible srqe->flags in bnxt_qplib_post_srq_recv(). Most values of the wqe struct is set except flags. So initialize flags also. Fixes: 37cb11acf1f7 ("RDMA/bnxt_re: Add SRQ support for Broadcom adapters") Signed-off-by: Selvin Xavier Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/ib_verbs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 17bcaad51a89..ac15372376a2 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -2397,6 +2397,7 @@ int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, spin_lock_irqsave(&srq->lock, flags); while (wr) { /* Transcribe each ib_recv_wr to qplib_swqe */ + wqe.flags = 0; wqe.num_sge = wr->num_sge; bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); wqe.wr_id = wr->wr_id; From 81caf9240dfb082c0eb2b24feef8266268fd152f Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 21 Sep 2026 23:12:28 -0700 Subject: [PATCH 081/153] RDMA/bnxt_re: Serialize dcb_wq access against async notifier bnxt_re_async_notifier() runs in NAPI/softirq context via bnxt_en's RCU-protected ULP ops dispatch, never under rtnl_lock, so it isn't serialized against bnxt_re_uninit_dcb_wq() destroying rdev->dcb_wq. A concurrent notifier call can queue_work() on a workqueue that is being, or has just been, destroyed. Add a spinlock scoped to dcb_wq: the notifier takes it before checking dcb_wq and queuing work, and bnxt_re_uninit_dcb_wq() takes it to atomically clear dcb_wq before destroying it. Initialize the lock at rdev allocation so it is valid on every teardown path, including bnxt_re_dev_init()'s early failure labels. Fixes: 51dc5312dcd9 ("RDMA/bnxt_re: Add support to handle DCB_CONFIG_CHANGE event") Signed-off-by: Selvin Xavier Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/bnxt_re.h | 4 ++++ drivers/infiniband/hw/bnxt_re/main.c | 26 ++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h index a43e678151d3..da471c63a3ba 100644 --- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h @@ -216,6 +216,10 @@ struct bnxt_re_dev { unsigned long event_bitmap; struct bnxt_qplib_cc_param cc_param; struct workqueue_struct *dcb_wq; + /* Protects dcb_wq against bnxt_re_uninit_dcb_wq() destroying it + * concurrently with bnxt_re_async_notifier() queuing work on it. + */ + spinlock_t dcb_lock; struct dentry *cc_config; struct bnxt_re_dbg_cc_config_params *cc_config_params; struct dentry *cq_coal_cfg; diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index ce72db1b4bc3..f7d23b11a07a 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -363,9 +363,15 @@ static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) { - if (!rdev->dcb_wq) - return; - destroy_workqueue(rdev->dcb_wq); + struct workqueue_struct *dcb_wq; + + spin_lock_bh(&rdev->dcb_lock); + dcb_wq = rdev->dcb_wq; + rdev->dcb_wq = NULL; + spin_unlock_bh(&rdev->dcb_lock); + + if (dcb_wq) + destroy_workqueue(dcb_wq); } static void bnxt_re_dcb_wq_task(struct work_struct *work) @@ -420,14 +426,23 @@ static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *c switch (event_id) { case ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: - dcb_work = kzalloc_obj(*dcb_work, GFP_ATOMIC); - if (!dcb_work) + spin_lock(&rdev->dcb_lock); + if (!rdev->dcb_wq) { + spin_unlock(&rdev->dcb_lock); break; + } + + dcb_work = kzalloc_obj(*dcb_work, GFP_ATOMIC); + if (!dcb_work) { + spin_unlock(&rdev->dcb_lock); + break; + } dcb_work->rdev = rdev; memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl)); INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task); queue_work(rdev->dcb_wq, &dcb_work->work); + spin_unlock(&rdev->dcb_lock); break; default: break; @@ -1444,6 +1459,7 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct auxiliary_device *adev, INIT_LIST_HEAD(&rdev->qp_list); mutex_init(&rdev->qp_lock); mutex_init(&rdev->pacing.dbq_lock); + spin_lock_init(&rdev->dcb_lock); atomic_set(&rdev->stats.res.qp_count, 0); atomic_set(&rdev->stats.res.cq_count, 0); atomic_set(&rdev->stats.res.srq_count, 0); From aa6c2ef821a6ba7f3fd58cfcc031b2d16f3811c6 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 21 Sep 2026 23:12:29 -0700 Subject: [PATCH 082/153] RDMA/bnxt_re: Fix the PD and DPI table size The PD and DPI bitmaps were sized as max >> 3 (bytes), but bitmap ops (set_bit(), clear_bit(), find_first_bit(), test_and_set_bit()) operate on whole unsigned long words, so whenever max isn't a multiple of BITS_PER_LONG, the buffer under-allocates and the top word's bitops read/write past the end of the kmalloc()'d buffer. Most exposed on the DPI table, since dpit->max comes from the firmware-reported dev_attr->max_dpi with no alignment guarantee. Fix the size of both allocations with BITS_TO_LONGS(max) * sizeof(unsigned long). Fixes: 1ac5a4047975 ("RDMA/bnxt_re: Add bnxt_re RoCE driver") Signed-off-by: Selvin Xavier Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/qplib_res.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 756f8b5f042a..7ff587ce9126 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -668,9 +669,9 @@ static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, { u32 bytes; - bytes = max >> 3; + bytes = BITS_TO_LONGS(max) * sizeof(unsigned long); if (!bytes) - bytes = 1; + bytes = sizeof(unsigned long); pdt->tbl = kmalloc(bytes, GFP_KERNEL); if (!pdt->tbl) return -ENOMEM; @@ -848,9 +849,9 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, if (!dpit->app_tbl) return -ENOMEM; - bytes = dpit->max >> 3; + bytes = BITS_TO_LONGS(dpit->max) * sizeof(unsigned long); if (!bytes) - bytes = 1; + bytes = sizeof(unsigned long); dpit->tbl = kmalloc(bytes, GFP_KERNEL); if (!dpit->tbl) { From 7cc8ccb157a4769c8fbce8296532dcb8a8ea0d5e Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Mon, 21 Sep 2026 23:12:30 -0700 Subject: [PATCH 083/153] RDMA/bnxt_re: Use bnxt_ext_stats_supported for counter count selection Selecting between BNXT_RE_NUM_EXT_COUNTERS and BNXT_RE_NUM_STD_COUNTERS relied only on bnxt_qplib_is_chip_gen_p5_p7(), which doesn't account for the extended stats capability flag or the PF/VF restriction. Use bnxt_ext_stats_supported() instead, matching the check already used to populate the extended stats, so the counter count stays consistent with what gets filled in. Fixes: 8238c7bd8420 ("RDMA/bnxt_re: Fix the statistics for Gen P7 VF") Reviewed-by: Kalesh AP Signed-off-by: Selvin Xavier Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/hw_counters.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c index 651cf9d0e0c7..575b97030587 100644 --- a/drivers/infiniband/hw/bnxt_re/hw_counters.c +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c @@ -413,7 +413,8 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, } done: - return bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? + return bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn) ? BNXT_RE_NUM_EXT_COUNTERS : BNXT_RE_NUM_STD_COUNTERS; } @@ -423,7 +424,8 @@ struct rdma_hw_stats *bnxt_re_ib_alloc_hw_port_stats(struct ib_device *ibdev, struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); int num_counters = 0; - if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn)) num_counters = BNXT_RE_NUM_EXT_COUNTERS; else num_counters = BNXT_RE_NUM_STD_COUNTERS; From 90e9b6357bec75f627cde1a85b5887215c80aad8 Mon Sep 17 00:00:00 2001 From: Dairui Zhang Date: Tue, 22 Sep 2026 23:23:42 +0800 Subject: [PATCH 084/153] IB/mad: fix UAF and double-free in RMPP receive reassembly mad_rmpp_recv.state is protected by two locks that do not exclude each other: recv_timeout_handler() (port workqueue) uses agent->lock, continue_rmpp() (CQ softirq) uses rmpp_recv->lock. Nothing serializes the check-then-set sequence on the state word: T1 (CQ softirq) T2 (port workqueue) continue_rmpp() lock(rmpp_recv->lock) state == ACTIVE [passes] recv_timeout_handler() lock(agent->lock) state = TIMEOUT list_del(&rmpp_recv->list) unlock(agent->lock) destroy_rmpp_recv() wait_for_completion() [blocks on T1's ref] state = COMPLETE unlock(rmpp_recv->lock) complete_rmpp() queue cleanup_work (+10s) deref() [unblocks] kfree(rmpp_recv) ib_free_recv_mad(done_wc) +10s: recv_cleanup_handler() runs lock/list_del/destroy on the freed rmpp_recv -> UAF write and double free; nothing can cancel that cleanup_work anymore, the entry is already off agent->rmpp_list. done_wc is also returned to ib_mad_complete_recv() and freed again -> UAF read and double free. Reaching this requires sending MADs to a kernel RMPP agent from the InfiniBand/RoCE fabric: the per-port sa_query agent accepts subnet manager responses whose SLID/GID/TID are not authenticated on the fabric, and legacy user_mad agents registered without IB_USER_MAD_USER_RMPP start reassembly on any incoming RMPP DATA MAD with an attacker-chosen TID, no in-flight send required. The attacker controls segment timing and can run many transactions in parallel (agent->rmpp_list has no cap), so landing the microseconds-wide receive critical section on the 40-second timeout boundary is a matter of patience, not luck. Unify state transitions under rmpp_recv->lock in recv_timeout_handler(), keeping list manipulation under agent->lock (which preserves the agent->lock -> rmpp_recv->lock ordering used by continue_rmpp()). State check/set in both paths is then serialized, so the loser of the race always observes the final state and bails out. recv_cleanup_handler() needs no change: with transitions serialized, a cleanup_work can only be pending after COMPLETE, in which case the timeout handler exits early. The race is as old as the RMPP implementation itself. Fixes: fa619a77046b ("[PATCH] IB: Add RMPP implementation") Assisted-by: LLM Signed-off-by: Dairui Zhang Link: https://patch.msgid.link/20260922152342.1474845-1-zhangdairui@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/mad_rmpp.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c index 17c4c52a19e4..f05e9ce93c33 100644 --- a/drivers/infiniband/core/mad_rmpp.c +++ b/drivers/infiniband/core/mad_rmpp.c @@ -245,12 +245,15 @@ static void recv_timeout_handler(struct work_struct *work) struct ib_mad_recv_wc *rmpp_wc; unsigned long flags; - spin_lock_irqsave(&rmpp_recv->agent->lock, flags); + spin_lock_irqsave(&rmpp_recv->lock, flags); if (rmpp_recv->state != RMPP_STATE_ACTIVE) { - spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags); + spin_unlock_irqrestore(&rmpp_recv->lock, flags); return; } rmpp_recv->state = RMPP_STATE_TIMEOUT; + spin_unlock_irqrestore(&rmpp_recv->lock, flags); + + spin_lock_irqsave(&rmpp_recv->agent->lock, flags); list_del(&rmpp_recv->list); spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags); From b840f0890ad06f2e79ad5a47e3283d2f46dc8a4c Mon Sep 17 00:00:00 2001 From: Dana Malachi Date: Tue, 22 Sep 2026 16:01:17 +0000 Subject: [PATCH 085/153] RDMA/efa: Use device ABI MR permissions instead of verbs flags Set MR permissions explicitly using device interface definitions rather than copying raw verbs access flags. This is needed for the next commit, to allow access flag bits that are not in the permissions field. Reviewed-by: Chen Brasch Reviewed-by: Yonatan Nachum Signed-off-by: Dana Malachi Signed-off-by: Michael Margolin Link: https://patch.msgid.link/20260922160118.990-2-mrgolin@amazon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/efa/efa_com_cmd.c | 7 ++++++- drivers/infiniband/hw/efa/efa_com_cmd.h | 7 +++++-- drivers/infiniband/hw/efa/efa_verbs.c | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index 1b00f16b8ea8..aac0e936601a 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -237,7 +237,12 @@ int efa_com_register_mr(struct efa_com_dev *edev, EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT, params->page_shift); mr_cmd.iova = params->iova; - mr_cmd.permissions = params->permissions; + if (params->permissions.local_write) + EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE, 1); + if (params->permissions.remote_write) + EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE, 1); + if (params->permissions.remote_read) + EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE, 1); if (params->inline_pbl) { memcpy(mr_cmd.pbl.inline_pbl_array, diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h index bca722f021ea..7b0399811cfd 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.h +++ b/drivers/infiniband/hw/efa/efa_com_cmd.h @@ -204,8 +204,11 @@ struct efa_com_reg_mr_params { * address mapping */ u8 page_shift; - /* see permissions field of struct efa_admin_reg_mr_cmd */ - u8 permissions; + struct { + u8 local_write : 1; + u8 remote_write : 1; + u8 remote_read : 1; + } permissions; u8 inline_pbl; u8 indirect; }; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 2d28d68efe77..635687275350 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1676,7 +1676,9 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start, params.pd = to_epd(ibpd)->pdn; params.iova = virt_addr; params.mr_length_in_bytes = length; - params.permissions = access_flags; + params.permissions.local_write = !!(access_flags & IB_ACCESS_LOCAL_WRITE); + params.permissions.remote_write = !!(access_flags & IB_ACCESS_REMOTE_WRITE); + params.permissions.remote_read = !!(access_flags & IB_ACCESS_REMOTE_READ); pg_sz = ib_umem_find_best_pgsz(mr->umem, dev->dev_attr.page_size_cap, From eb6aed55cc01521608bea0ae57fb25984fa1162e Mon Sep 17 00:00:00 2001 From: Dana Malachi Date: Tue, 22 Sep 2026 16:01:18 +0000 Subject: [PATCH 086/153] RDMA/efa: Pass relaxed ordering flag to device Accept the relaxed ordering access flag during memory region registration and forward it to the device firmware via the admin command path. Reviewed-by: Chen Brasch Reviewed-by: Yonatan Nachum Signed-off-by: Dana Malachi Signed-off-by: Michael Margolin Link: https://patch.msgid.link/20260922160118.990-3-mrgolin@amazon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/efa/efa_admin_cmds_defs.h | 5 +++-- drivers/infiniband/hw/efa/efa_com_cmd.c | 2 ++ drivers/infiniband/hw/efa/efa_com_cmd.h | 1 + drivers/infiniband/hw/efa/efa_verbs.c | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index ab830764e3b4..30a38e633bfd 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -358,7 +358,8 @@ struct efa_admin_reg_mr_cmd { * 5:0 : phys_page_size_shift - page size is (1 << * phys_page_size_shift). Page size is used for * building the Virtual to Physical address mapping - * 6 : reserved - MBZ + * 6 : relaxed_ordering_enable - Enable PCIe relaxed + * ordering for this memory region * 7 : mem_addr_phy_mode_en - Enable bit for physical * memory registration (no translation), can be used * only by privileged clients. If set, PBL must @@ -1203,6 +1204,7 @@ struct efa_admin_modify_event_counter_resp { /* reg_mr_cmd */ #define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(5, 0) +#define EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE_MASK BIT(6) #define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK BIT(7) #define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK BIT(0) #define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK BIT(1) @@ -1231,7 +1233,6 @@ struct efa_admin_modify_event_counter_resp { #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK BIT(3) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_EVENT_COUNTERS_MASK BIT(5) -#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_SHIFT 10 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_MASK BIT(10) /* feature_queue_attr_desc_2 */ diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index aac0e936601a..07bfcb093df8 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -236,6 +236,8 @@ int efa_com_register_mr(struct efa_com_dev *edev, mr_cmd.mr_length = params->mr_length_in_bytes; EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT, params->page_shift); + if (params->relaxed_ordering) + EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE, 1); mr_cmd.iova = params->iova; if (params->permissions.local_write) EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE, 1); diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h index 7b0399811cfd..524407f36d43 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.h +++ b/drivers/infiniband/hw/efa/efa_com_cmd.h @@ -209,6 +209,7 @@ struct efa_com_reg_mr_params { u8 remote_write : 1; u8 remote_read : 1; } permissions; + u8 relaxed_ordering; u8 inline_pbl; u8 indirect; }; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 635687275350..9e4435401a4c 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1679,6 +1679,7 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start, params.permissions.local_write = !!(access_flags & IB_ACCESS_LOCAL_WRITE); params.permissions.remote_write = !!(access_flags & IB_ACCESS_REMOTE_WRITE); params.permissions.remote_read = !!(access_flags & IB_ACCESS_REMOTE_READ); + params.relaxed_ordering = !!(access_flags & IB_ACCESS_RELAXED_ORDERING); pg_sz = ib_umem_find_best_pgsz(mr->umem, dev->dev_attr.page_size_cap, From 9f4c322be27bf617cad2f4a15f6c6024a69e220c Mon Sep 17 00:00:00 2001 From: Yili Zhang Date: Wed, 23 Sep 2026 21:12:39 +0800 Subject: [PATCH 087/153] RDMA/nldev: Fix NULL deref in dumps of objects abandoned by DRIVER_FAILURE fill_res_cq_entry() dereferences cq->uobject->uevent.uobject.context->res.id unconditionally for user resources. This is normally safe by ordering: destroy_hw() removes the resource from the restrack before uverbs_destroy_uobject() clears ->context, and the XA_ZERO_ENTRY marker set by rdma_restrack_begin_del() hides the entry from concurrent netlink dumps. The ordering breaks when a driver keeps failing to destroy an object during ucontext teardown. uverbs_destroy_ufile_hw() then falls back to __uverbs_cleanup_ufile(RDMA_REMOVE_DRIVER_FAILURE), which abandons the object in place: the HW object and its restrack entry are intentionally leaked, while the uobject bookkeeping is torn down and ->context is explicitly set to NULL by uverbs_destroy_uobject(). rdma_restrack_del() is never reached, so the orphaned entry stays in the device restrack with a valid kref, reachable by any subsequent netlink dump. Observed on 6.1.52 with MLNX_OFED 24.10, after an mlx5 FW failure left a process unable to tear down its CQ (destroy_cq FW command failing during FD close): WARNING: ... uverbs_destroy_ufile_hw+0xe3/0x100 BUG: kernel NULL pointer dereference, address: 0000000000000058 RIP: 0010:fill_res_cq_entry+0x15e/0x180 [ib_core] res_get_common_dumpit+0x304/0x530 [ib_core] nldev_res_get_cq_dumpit+0x1a/0x20 [ib_core] The faulting chain maps to the source (CR2 = 0x58): cq->uobject (struct ib_cq +0x08, res at +0x98) uobject->context (struct ib_uobject +0x10, NULL after abandon) context->res.id (struct ib_ucontext +0x58) The recent restrack rework (8d186210677c and its series) moved the restrack deletion to the start of the destroy flow and thus fences concurrent dumps from an object being destroyed, but it does not cover this case: when destroy fails, rdma_restrack_abort_del() restores the entry, and the RDMA_REMOVE_DRIVER_FAILURE sweep still never removes it from the restrack. From the fallback until the device is unregistered, any "rdma res show cq" deterministically takes the NULL pointer dereference; this is a long-lived state, NOT A RACE. The dump path holds neither the restrack lock (dropped before the fill callback runs) nor ufile->hw_destroy_rwsem, and rdma_restrack_get() only guarantees that the res memory stays alive, not that ->context is still valid. Fix the dump side: check the uobject context directly instead of the rdma_is_kernel_res() marker. A non-NULL uobject already implies a user resource, so kernel resources keep being skipped, and the additional ->context check skips the RES_CTXN attribute for orphaned entries instead of crashing the dump. The orphaned resource itself remains visible in "rdma res show" (cqn/cqe/usecnt/pid), which is what an operator needs after the accompanying uverbs WARN to diagnose the driver destroy failure. fill_res_pd_entry() has the same pattern (pd->uobject->context->res.id) and is fixed the same way; a PD can even reach the fallback without its own driver callback failing, e.g. uverbs_free_pd() returns -EBUSY while another object that failed to destroy still holds the PD usecnt. Fixes: c3d02788b45a ("RDMA/nldev: Provide parent IDs for PD, MR and QP objects") Signed-off-by: Yili Zhang Link: https://patch.msgid.link/20260923131239.6521-1-zhangyili01@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/nldev.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index a1542ed80fcc..de292930388f 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -700,7 +700,8 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin, if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id)) return -EMSGSIZE; - if (!rdma_is_kernel_res(res) && + + if (cq->uobject && cq->uobject->uevent.uobject.context && nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, cq->uobject->uevent.uobject.context->res.id)) return -EMSGSIZE; @@ -790,7 +791,7 @@ static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin, if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id)) goto err; - if (!rdma_is_kernel_res(res) && + if (pd->uobject && pd->uobject->context && nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, pd->uobject->context->res.id)) goto err; From 724cd362162ca7ca5a250bec65efd74f7dad9f5c Mon Sep 17 00:00:00 2001 From: Quanye Yang Date: Tue, 22 Sep 2026 21:09:58 -0700 Subject: [PATCH 088/153] RDMA/rtrs-clt: Reject invalid queue_depth from the peer queue_depth is taken from the server's CM private_data and later used as a session invariant. A peer can send 0, which the client stores and then trips WARN_ON() in create_con_cq_qp() when the next connection is set up. Reject a zero queue_depth on ESTABLISHED, same as a bad magic or version, and fail the handshake with -ECONNRESET. Fixes: 6a98d71daea1 ("RDMA/rtrs: client: main functionality") Reported-by: Farhad Alemi Closes: https://lore.kernel.org/linux-rdma/CA+0ovCg_sG8gaZXXj9zmOrpxRt=8+-x9wvycVNUKwWmMbr6-Yg@mail.gmail.com Signed-off-by: Quanye Yang Link: https://patch.msgid.link/20260922-rtrs-fix-warning-rtrs-clt-rdma-cm-handler-v1-1-2076c0ee1455@proton.me Acked-by: Jack Wang Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index f48872bb0024..447aaa28fd5d 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -1852,6 +1852,10 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, } if (con->c.cid == 0) { queue_depth = le16_to_cpu(msg->queue_depth); + if (!queue_depth) { + rtrs_err(clt, "Invalid queue depth %u\n", queue_depth); + return -ECONNRESET; + } if (clt_path->queue_depth > 0 && queue_depth != clt_path->queue_depth) { rtrs_err(clt, "Error: queue depth changed\n"); From 4af07308d8227affa1710b392ba79af7e878073e Mon Sep 17 00:00:00 2001 From: Binbin Deng <18983559317@163.com> Date: Wed, 23 Sep 2026 13:18:00 +0800 Subject: [PATCH 089/153] RDMA/rxe: unpublish the per-net tunnel socket before KASAN reports a slab-use-after-free in ip6_route_output_flags() reached from rxe_find_route(), with the free in __sk_destruct() after rxe_sock_put(), and a user with CAP_NET_ADMIN can remove the device with "rdma link del rxe0" while RoCE v2 over IPv6 traffic keeps looking the socket up. BUG: KASAN: slab-use-after-free in ip6_route_output_flags+0x300/0x360 Read of size 4 at addr ffff888115ddd794 by task kworker/u32:6/309 Call Trace: dump_stack_lvl+0x53/0x70 print_report+0xd0/0x630 ? __pfx__raw_spin_lock_irqsave+0x10/0x10 ? ip6_route_output_flags+0x300/0x360 kasan_report+0xce/0x100 ? ip6_route_output_flags+0x300/0x360 ip6_route_output_flags+0x300/0x360 ip6_dst_lookup_tail.constprop.0+0x76c/0xcc0 ? ct_nmi_exit+0xc3/0xf0 ip6_dst_lookup_flow+0xf5/0x1e0 ? __pfx_ip6_dst_lookup_flow+0x10/0x10 rxe_find_route+0x426/0xa30 ? __kasan_slab_alloc+0x6e/0x70 ? __pfx_rxe_find_route+0x10/0x10 ? kmem_cache_alloc_node_noprof+0x141/0x370 ? kmalloc_reserve+0x103/0x2b0 ? rxe_icrc_generate+0x229/0x330 ? __pfx___alloc_skb+0x10/0x10 rxe_prepare+0x9e8/0x18b0 ? rxe_init_packet+0x3c7/0x4f0 rxe_requester+0x1a0f/0x51f0 ? rxe_completer+0x1de9/0x38c0 ? __pfx_rxe_completer+0x10/0x10 ? __queue_work+0x43e/0x11f0 ? __pfx_rxe_requester+0x10/0x10 ? irqentry_exit+0xd2/0x640 ? _raw_spin_lock_irqsave+0x85/0xe0 ? __pfx__raw_spin_lock_irqsave+0x10/0x10 ? __pfx_rxe_sender+0x10/0x10 rxe_sender+0xe/0x30 do_work+0x144/0x470 process_one_work+0x633/0x1030 ? assign_work+0x11d/0x370 worker_thread+0x45b/0xd10 ? __pfx_worker_thread+0x10/0x10 kthread+0x2c6/0x3b0 ? recalc_sigpending+0x15c/0x1e0 ? __pfx_kthread+0x10/0x10 ret_from_fork+0x36e/0x5a0 ? __pfx_ret_from_fork+0x10/0x10 ? __switch_to+0x572/0xdd0 ? __pfx_kthread+0x10/0x10 ret_from_fork_asm+0x1a/0x30 Allocated by task 146020: kasan_save_stack+0x33/0x60 kasan_save_track+0x14/0x30 __kasan_slab_alloc+0x6e/0x70 kmem_cache_alloc_noprof+0x130/0x360 sk_prot_alloc+0x56/0x210 Fix by clearing the per-net pointer before the last reference is dropped. Fixes: f1327abd6abed ("RDMA/rxe: Support RDMA link creation and destruction per net namespace") Signed-off-by: Binbin Deng <18983559317@163.com> Link: https://patch.msgid.link/20260923051800.244740-1-18983559317@163.com Reviewed-by: Zhu Yanjun Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_net.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index c66ae71d970d..9f1c7f21d88d 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -672,9 +672,8 @@ static void rxe_sock_put(struct sock *sk, if (refcount_read(&sk->sk_refcnt) > SK_REF_FOR_TUNNEL) { __sock_put(sk); } else { + set_sk(net, NULL); rxe_release_udp_tunnel(sk); - sk = NULL; - set_sk(net, sk); } } From 2dac7006b9fb466a378c2018b44496e62f68b427 Mon Sep 17 00:00:00 2001 From: Norbert Szetei Date: Wed, 23 Sep 2026 16:40:49 +0200 Subject: [PATCH 090/153] RDMA/rxe: Reject IB_ACCESS_ON_DEMAND changes after MR creation Whether an MR is an ODP MR is decided once, at registration time: rxe_reg_user_mr() picks rxe_odp_mr_init_user() over rxe_mr_init_user() based on IB_ACCESS_ON_DEMAND, and only the former builds an ib_umem_odp via ib_umem_odp_get(). The umem cannot change type afterwards, and is_odp_mr() reads mr->umem->is_odp. Two paths assign mr->access after that point and can leave it describing an MR type the umem does not have: rxe_rereg_user_mr() with IB_MR_REREG_ACCESS overwrites mr->access with the caller's value, and IB_ACCESS_ON_DEMAND is part of RXE_ACCESS_SUPPORTED_MR, so userspace can set the flag on a plain MR or clear it on an ODP MR while the umem stays what it was. rxe_reg_fast_mr() takes mr->access from the REG_MR work request unmasked and moves the MR to RXE_MR_STATE_VALID, on an MR that rxe_mr_init_fast() left with a NULL umem. Reject IB_ACCESS_ON_DEMAND in both, so mr->access carries the flag only for an MR that has an ODP umem and the flag can be used to identify one. Fixes: 544c7f62cf32 ("RDMA/rxe: Implement rereg_user_mr") Signed-off-by: Norbert Szetei Link: https://patch.msgid.link/20260923-rxe-advise-mr-v3-v3-1-95e1a4077e4d@doyensec.com Reviewed-by: Zhu Yanjun Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_mr.c | 6 ++++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index 875eceb55fdf..2afda5154dfb 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -795,6 +795,12 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe) return -EINVAL; } + /* an MR with no umem is never an ODP MR */ + if (unlikely(access & IB_ACCESS_ON_DEMAND)) { + rxe_dbg_mr(mr, "access = 0x%x requests ODP\n", access); + return -EINVAL; + } + mr->access = access; mr->lkey = key; mr->rkey = key; diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 96c7716057fe..21855274f63a 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -1331,6 +1331,12 @@ static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags, if (err) return ERR_PTR(err); + if ((flags & IB_MR_REREG_ACCESS) && + ((access ^ mr->access) & IB_ACCESS_ON_DEMAND)) { + rxe_err_mr(mr, "cannot change IB_ACCESS_ON_DEMAND\n"); + return ERR_PTR(-EOPNOTSUPP); + } + if (flags & IB_MR_REREG_PD) { rxe_put(old_pd); rxe_get(pd); From 66f18a749a3eeeb19f0b7812ecbe8b1ac18d73ec Mon Sep 17 00:00:00 2001 From: Norbert Szetei Date: Wed, 23 Sep 2026 16:40:50 +0200 Subject: [PATCH 091/153] RDMA/rxe: Reject prefetch of a non-ODP MR rxe_ib_advise_mr_prefetch() and rxe_ib_prefetch_sg_list() look up the MR by lkey and hand it to rxe_odp_do_pagefault_and_lock() without checking that it is an ODP MR. That path runs to_ib_umem_odp() on mr->umem, and for a non-ODP MR mr->umem is a plain struct ib_umem from ib_umem_get(), so the container_of() in to_ib_umem_odp() lands past the end of the object and ib_umem_odp_map_dma_and_lock() reads its ib_umem_odp fields out of bounds. lookup_mr() validates the lkey, PD, access and state but not the MR type, and IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH is accepted for any MR. BUG: KASAN: slab-out-of-bounds in ib_umem_odp_map_dma_and_lock+0x884/0x8a0 Read of size 8 at addr ffff88810a3ebcf0 by task advi/921 ib_umem_odp_map_dma_and_lock+0x884/0x8a0 rxe_ib_advise_mr+0x543/0xad0 ib_uverbs_handler_UVERBS_METHOD_ADVISE_MR+0x446/0x530 ib_uverbs_cmd_verbs+0x2b3c/0x3b20 ib_uverbs_ioctl+0x1e3/0x310 Allocated by task 921: __ib_umem_get_va+0x13e/0xae0 rxe_mr_init_user+0x2ae/0xb00 rxe_reg_user_mr+0x337/0x510 The buggy address belongs to the object at ffff88810a3ebc80 which belongs to the cache kmalloc-96 of size 96 Ask lookup_mr() for IB_ACCESS_ON_DEMAND in both the synchronous and the asynchronous prefetch arm. mr->access carries that flag only for an MR registered as ODP, so the existing (access & mr->access) != access test rejects a plain MR and the prefetch fails with -EINVAL. Fixes: 3576b0df1588 ("RDMA/rxe: Implement synchronous prefetch for ODP MRs") Signed-off-by: Norbert Szetei Link: https://patch.msgid.link/20260923-rxe-advise-mr-v3-v3-2-95e1a4077e4d@doyensec.com Reviewed-by: Zhu Yanjun Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_odp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c index e870efa7a0a3..f8ffd6a3219a 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -463,7 +463,7 @@ static int rxe_ib_prefetch_sg_list(struct ib_pd *ibpd, struct rxe_mr *mr; struct ib_umem_odp *umem_odp; - mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE, + mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE | IB_ACCESS_ON_DEMAND, sg_list[i].lkey, RXE_LOOKUP_LOCAL); if (!mr) { @@ -529,7 +529,7 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd, for (i = 0; i < num_sge; ++i) { /* Takes a reference, which will be released in the queued work */ - mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE, + mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE | IB_ACCESS_ON_DEMAND, sg_list[i].lkey, RXE_LOOKUP_LOCAL); if (!mr) { mr = ERR_PTR(-EINVAL); From ad91fd9e9440ab23b3716aaac332e5d237535afb Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Thu, 24 Sep 2026 15:39:19 +0800 Subject: [PATCH 092/153] RDMA/siw: drop duplicate check_app_limited in siw_tcp_sendpages After do_tcp_sendpages() was converted to use MSG_SPLICE_PAGES and later inlined as direct tcp_sendmsg_locked() calls, callers that previously needed an explicit tcp_rate_check_app_limited(sk) to cover do_tcp_sendpages() no longer need it. tcp_sendmsg_locked() performs the check on every path that queues data, making the outer call redundant. The site changed here, siw_tcp_sendpages() in siw_qp_tx.c, holds the socket lock and invokes tcp_sendmsg_locked() on every iteration. The early-return paths in tcp_sendmsg_locked() that skip tcp_rate_check_app_limited() - the MSG_ZEROCOPY allocation failure and MSG_FASTOPEN branches - return without queueing any MSG_SPLICE_PAGES data, so there is no functional consequence from omitting the outer check. Signed-off-by: Geliang Tang Link: https://patch.msgid.link/32ef40cc81486adb8f2d43585224ab44d7e77f73.1790234916.git.tanggeliang@kylinos.cn Acked-by: Bernard Metzler Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/siw/siw_qp_tx.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c index f7dd32c6e5ba..56bd009879c2 100644 --- a/drivers/infiniband/sw/siw/siw_qp_tx.c +++ b/drivers/infiniband/sw/siw/siw_qp_tx.c @@ -336,7 +336,6 @@ static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset, if (size + offset <= PAGE_SIZE) msg.msg_flags &= ~MSG_MORE; - tcp_rate_check_app_limited(sk); if (!sendpage_ok(page[i])) msg.msg_flags &= ~MSG_SPLICE_PAGES; bvec_set_page(&bvec, page[i], bytes, offset); From 2eca9c14e66a1f71c089fe5c3afb2d339da3023a Mon Sep 17 00:00:00 2001 From: Abhijit Gangurde Date: Fri, 25 Sep 2026 15:06:37 +0530 Subject: [PATCH 093/153] RDMA/ionic: Fix XArray initialization to use XArray flags xa_init_flags() takes XArray flags (XA_FLAGS_*), not gfp flags, even though the parameter is typed gfp_t. Passing GFP_ATOMIC leaves both lock-type bits clear, so xa_lock_type() resolves to XA_LOCK_NORMAL for qp_tbl and cq_tbl. Both tables are used with the IRQ-safe accessors: xa_store_irq() with GFP_KERNEL from process context, and xa_lock_irqsave() from the event queue path. When a store needs to grow the tree, __xas_nomem() drops the lock using the recorded lock type before allocating: if (gfpflags_allow_blocking(gfp)) { xas_unlock_type(xas, lock_type); xas->xa_alloc = kmem_cache_alloc_lru(...); xas_lock_type(xas, lock_type); } With XA_LOCK_NORMAL that is a plain spin_unlock(), so the sleeping GFP_KERNEL allocation runs with interrupts still disabled by xa_store_irq(), and lockdep annotates the lock with the wrong class. GFP_ATOMIC also happens to set __GFP_HIGH (0x20), which collides with XA_FLAGS_ACCOUNT (32U) and silently enables memcg accounting of the XArray nodes. Use XA_FLAGS_LOCK_IRQ so the lock type matches how the tables are actually accessed. Fixes: e8521822c733 ("RDMA/ionic: Register device ops for control path") Signed-off-by: Abhijit Gangurde Link: https://patch.msgid.link/20260925093638.3032149-1-abhijit.gangurde@amd.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/ionic/ionic_ibdev.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index cba7809ec3d9..fa79732c2f7d 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -320,8 +320,8 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg); - xa_init_flags(&dev->qp_tbl, GFP_ATOMIC); - xa_init_flags(&dev->cq_tbl, GFP_ATOMIC); + xa_init_flags(&dev->qp_tbl, XA_FLAGS_LOCK_IRQ); + xa_init_flags(&dev->cq_tbl, XA_FLAGS_LOCK_IRQ); ionic_init_resids(dev); From 485effd117d0310a7f589defb4f442fa3bb52ae2 Mon Sep 17 00:00:00 2001 From: Abhijit Gangurde Date: Fri, 25 Sep 2026 15:06:38 +0530 Subject: [PATCH 094/153] RDMA/ionic: Fix double removal of CMB mmap entries in create QP error path The error unwind in ionic_create_qp() removes the SQ and RQ CMB mmap entries explicitly, and then falls through to ionic_qp_rq_destroy() and ionic_qp_sq_destroy(), whose ionic_qp_{rq,sq}_destroy_cmb() helpers remove the very same entries a second time. Drop the explicit removals and leave ionic_qp_{rq,sq}_destroy_cmb() as the single owner on the ionic_destroy_qp() path. Fixes: e8521822c733 ("RDMA/ionic: Register device ops for control path") Signed-off-by: Abhijit Gangurde Link: https://patch.msgid.link/20260925093638.3032149-2-abhijit.gangurde@amd.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/ionic/ionic_controlpath.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index f392c635338b..2b68f377e1fa 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -2351,7 +2351,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, &resp.sq_cmb_offset); if (!qp->mmap_sq_cmb) { rc = -ENOMEM; - goto err_mmap_sq; + goto err_qp_cmd; } resp.sq_cmb = qp->sq_cmb; @@ -2388,7 +2388,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, &resp.rq_cmb_offset); if (!qp->mmap_rq_cmb) { rc = -ENOMEM; - goto err_mmap_rq; + goto err_qp_cmd; } resp.rq_cmb = qp->rq_cmb; @@ -2396,7 +2396,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, rc = ib_respond_udata(udata, resp); if (rc) - goto err_resp; + goto err_qp_cmd; } ionic_pgtbl_unbuf(dev, &rq_buf); @@ -2414,7 +2414,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, else rc = xa_err(entry); - goto err_resp; + goto err_qp_cmd; } if (qp->has_sq) { @@ -2445,13 +2445,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, return 0; -err_resp: - if (udata && (qp->rq_cmb & IONIC_CMB_ENABLE)) - rdma_user_mmap_entry_remove(qp->mmap_rq_cmb); -err_mmap_rq: - if (udata && (qp->sq_cmb & IONIC_CMB_ENABLE)) - rdma_user_mmap_entry_remove(qp->mmap_sq_cmb); -err_mmap_sq: +err_qp_cmd: ionic_destroy_qp_cmd(dev, qp->qpid); err_cmd: ionic_pgtbl_unbuf(dev, &rq_buf); From 2dc2126e690203d6adbddc7675927a242d810e43 Mon Sep 17 00:00:00 2001 From: Jiale Yao Date: Sat, 26 Sep 2026 18:59:25 +0800 Subject: [PATCH 095/153] RDMA/mad: Reject receive completions shorter than GRH ib_mad_recv_done() derives the OPA MAD length by subtracting the GRH size from wc->byte_len without ensuring the receive completion covers the GRH. If it is shorter, the subtraction wraps before the result is stored in mad_len and an invalid length can be passed downstream. Reject receive completions shorter than the GRH before inspecting the MAD header or deriving the MAD length. Fixes: 8e4349d13f33 ("IB/mad: Add final OPA MAD processing") Signed-off-by: Jiale Yao Link: https://patch.msgid.link/20260926105927.3183294-1-yaojiale02@163.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/mad.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index e0b3b36b8b14..bcc19d1fa036 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -2341,6 +2341,8 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc) recv->header.mapping, mad_priv_dma_size(recv), DMA_FROM_DEVICE); + if (unlikely(wc->byte_len < sizeof(struct ib_grh))) + goto out; /* Setup MAD receive work completion from "normal" work completion */ recv->header.wc = *wc; From 33db1d9683b8fb00bc233d0bea093e694ed9afc5 Mon Sep 17 00:00:00 2001 From: Jiale Yao Date: Sat, 26 Sep 2026 19:11:50 +0800 Subject: [PATCH 096/153] RDMA/mlx4: Do not allow the user to write to the clock page mlx4_ib_mmap() maps the internal clock page to userspace through rdma_user_mmap_io(), which does not restrict write access. Userspace can request PROT_WRITE directly, or upgrade a read-only mapping later with mprotect() because VM_MAYWRITE remains set. Reject VM_WRITE, VM_EXEC and clear VM_MAYWRITE, applying the same read-only policy as commit c660133c339f ("RDMA/mlx5: Do not allow the user to write to the clock page"). The rdma-core mlx4 provider requests this mapping with PROT_READ, so its existing behavior is unchanged. Fixes: 52033cfb5aab ("IB/mlx4: Add mmap call to map the hardware clock") Signed-off-by: Jiale Yao Link: https://patch.msgid.link/20260926111150.3197004-1-yaojiale02@163.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx4/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 7266a6141944..1594e6407b6c 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1169,6 +1169,10 @@ static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) struct mlx4_clock_params params; int ret; + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) + return -EPERM; + vm_flags_clear(vma, VM_MAYWRITE); + ret = mlx4_get_internal_clock_params(dev->dev, ¶ms); if (ret) return ret; From 1c5335a6902639596a795705349da7f76027e103 Mon Sep 17 00:00:00 2001 From: Jiale Yao Date: Sat, 26 Sep 2026 19:26:11 +0800 Subject: [PATCH 097/153] RDMA/cxgb4: Do not allow userspace to write the status page The device status page is shared by all userspace contexts and contains db_off, which the driver updates to control doorbell flow. The rdma-core provider maps it with PROT_READ, but c4iw_mmap() passes the requested page protection through when handling CXGB4_MMAP_CONTIG. A process can therefore request PROT_WRITE directly or later upgrade a read-only mapping with mprotect() because VM_MAYWRITE remains set. This lets userspace alter device-wide flow-control state seen by the kernel and other contexts. Reject initially writable mappings and clear VM_MAYWRITE before mapping the page. This preserves the existing read-only userspace ABI. Fixes: 05eb23893c2c ("cxgb4/iw_cxgb4: Doorbell Drop Avoidance Bug Fixes") Signed-off-by: Jiale Yao Link: https://patch.msgid.link/20260926112611.3214382-1-yaojiale02@163.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/cxgb4/provider.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index ebe3170a641c..904a5323a693 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -171,6 +171,11 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) len, t4_pgprot_wc(vma->vm_page_prot)); break; case CXGB4_MMAP_CONTIG: + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) { + ret = -EPERM; + break; + } + vm_flags_clear(vma, VM_MAYWRITE); ret = io_remap_pfn_range(vma, vma->vm_start, addr >> PAGE_SHIFT, len, vma->vm_page_prot); From 7e4d9504904f6e1ab9ea5bc2dfd1366c08483162 Mon Sep 17 00:00:00 2001 From: Michael Margolin Date: Thu, 17 Sep 2026 10:51:01 +0000 Subject: [PATCH 098/153] RDMA/efa: Use __GFP_RETRY_MAYFAIL for ring allocations RQ and CQ ring buffers require physically contiguous memory. On systems under memory pressure, high-order allocations can fail even when sufficient free memory exists because the allocator gives up after a single reclaim and compaction attempt for costly-order requests that lack __GFP_RETRY_MAYFAIL. Add __GFP_RETRY_MAYFAIL to the GFP flags in efa_zalloc_mapped() to allow retries with escalating compaction priority and release of highatomic reserves before giving up. Link: https://patch.msgid.link/r/20260917105101.9365-1-mrgolin@amazon.com Reviewed-by: Firas Jahjah Reviewed-by: Yonatan Nachum Signed-off-by: Michael Margolin Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/efa/efa_verbs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 9e4435401a4c..d56eb132545b 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -190,7 +190,7 @@ static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr, { void *addr; - addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); + addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL); if (!addr) return NULL; From 49c5691ad76f000e66c4f2635d5f196db5d038f9 Mon Sep 17 00:00:00 2001 From: Quanye Yang Date: Mon, 21 Sep 2026 21:25:25 -0700 Subject: [PATCH 099/153] RDMA/rtrs-clt: Don't WARN on unexpected info-response opcode The info-response receive is posted for a SEND. The completion opcode comes from the peer, so RDMA WRITE with Immediate is a legal WC on that WQE even though it violates the RTRS handshake. WARN_ON(wc->opcode != IB_WC_RECV) treats that peer-controlled field as an internal invariant and panics with panic_on_warn. Treat a non-SEND completion as a handshake failure: log it and keep the existing CONNECTING_ERR cleanup. Fixes: 6a98d71daea1 ("RDMA/rtrs: client: main functionality") Link: https://patch.msgid.link/r/20260921-rtrs-fix-rtrs-clt-info-rsp-done-v1-1-c64281ed97c0@proton.me Reported-by: Farhad Alemi Closes: https://lore.kernel.org/linux-rdma/CA+0ovCge-K+hcGHXZ4R3Ww5+cg159uxHBeVbKeUSLBRSMT8OWA@mail.gmail.com/ Signed-off-by: Quanye Yang Reviewed-by: Jack Wang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 447aaa28fd5d..8d6a4667422d 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -2515,7 +2515,12 @@ static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) ib_wc_status_msg(wc->status)); goto out; } - WARN_ON(wc->opcode != IB_WC_RECV); + if (wc->opcode != IB_WC_RECV) { + rtrs_err(clt_path->clt, + "Path info response has unexpected opcode %d\n", + wc->opcode); + goto out; + } if (wc->byte_len < sizeof(*msg)) { rtrs_err(clt_path->clt, "Path info response is malformed: size %d\n", From c1fcde82f7b68181553af7efb4410df04109a9a8 Mon Sep 17 00:00:00 2001 From: Quanye Yang Date: Wed, 23 Sep 2026 03:52:11 -0700 Subject: [PATCH 100/153] RDMA/rtrs-clt: Validate peer-supplied IO completion msg_id process_io_rsp() indexes clt_path->reqs[] with the 19-bit msg_id taken from a WRITE_WITH_IMM immediate. That value is chosen by the peer. The only check was WARN_ON(msg_id >= queue_depth), which still lets a completion run after free_path_reqs() has set reqs to NULL, or against a slot whose FRWR is already gone. Either path dereferences req->mr and can GPF; KASAN reports a null-ptr-deref at offsetof(struct rtrs_clt_io_req, mr). Treat a missing reqs array, an out-of-range msg_id, or a req without an MR as a protocol error and recover the connection. Link: https://patch.msgid.link/r/20260923-rtrs-warning-inrtrs-clt-rdma-done-v1-1-0827a9a77c0b@proton.me Reported-by: Farhad Alemi Closes: https://lore.kernel.org/r/CA+0ovCiJZz6O8LzsS9Dq1-JE1OE1L_OTA6Mr-WratpUE6jczHw@mail.gmail.com Fixes: 6a98d71daea1 ("RDMA/rtrs: client: main functionality") Signed-off-by: Quanye Yang Reviewed-by: Jack Wang Signed-off-by: Jason Gunthorpe --- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 8d6a4667422d..afd8875d9ede 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -494,18 +494,22 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con, imm, flags, wr, NULL); } -static void process_io_rsp(struct rtrs_clt_path *clt_path, u32 msg_id, +static bool process_io_rsp(struct rtrs_clt_path *clt_path, u32 msg_id, s16 errno, bool w_inval) { struct rtrs_clt_io_req *req; - if (WARN_ON(msg_id >= clt_path->queue_depth)) - return; + if (!clt_path->reqs || msg_id >= clt_path->queue_depth) + return false; req = &clt_path->reqs[msg_id]; + if (!req->mr) + return false; + /* Drop need_inv if server responded with send with invalidation */ req->mr->need_inval &= !w_inval; complete_rdma_req(req, errno, true, false); + return true; } static void rtrs_clt_recv_done(struct rtrs_clt_con *con, struct ib_wc *wc) @@ -567,7 +571,8 @@ static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc) if (WARN_ON(buf_id != msg_id)) goto out; clt_path->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey); - process_io_rsp(clt_path, msg_id, err, w_inval); + if (!process_io_rsp(clt_path, msg_id, err, w_inval)) + goto out; } ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev, iu->dma_addr, iu->size, DMA_FROM_DEVICE); @@ -639,7 +644,13 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM); rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err); - process_io_rsp(clt_path, msg_id, err, w_inval); + if (!process_io_rsp(clt_path, msg_id, err, w_inval)) { + rtrs_err(clt_path->clt, + "Invalid IO rsp: msg_id %u queue_depth %zu\n", + msg_id, clt_path->queue_depth); + rtrs_rdma_error_recovery(con); + return; + } } else if (imm_type == RTRS_HB_MSG_IMM) { WARN_ON(con->c.cid); rtrs_send_hb_ack(&clt_path->s); From aac27aa8c5cfe3c3c6029b9c0f1113e0a53329de Mon Sep 17 00:00:00 2001 From: Serhat Kumral Date: Sun, 27 Sep 2026 14:53:51 +0300 Subject: [PATCH 101/153] RDMA/siw: Fix length of first chunk in siw_try_1seg() When a short-single SGE payload crosses a page boundary, siw_try_1seg() computes the length of the first chunk as the amount that spills into the second page instead of the amount left in the first page part = bytes - (PAGE_SIZE - off); Use the remaining length of the first page as the first chunk. Fixes: b9be6f18cf9e ("rdma/siw: transmit path") Link: https://patch.msgid.link/r/20260927115351.12245-1-serhatkumral1@gmail.com Assisted-by: LLM Signed-off-by: Serhat Kumral Acked-by: Bernard Metzler Signed-off-by: Jason Gunthorpe --- drivers/infiniband/sw/siw/siw_qp_tx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c index 56bd009879c2..69fdcfffea80 100644 --- a/drivers/infiniband/sw/siw/siw_qp_tx.c +++ b/drivers/infiniband/sw/siw/siw_qp_tx.c @@ -85,7 +85,7 @@ static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr) if (likely(PAGE_SIZE - off >= bytes)) { memcpy(paddr, buffer + off, bytes); } else { - unsigned long part = bytes - (PAGE_SIZE - off); + unsigned long part = PAGE_SIZE - off; memcpy(paddr, buffer + off, part); kunmap_local(buffer); From f5d4cc88371feb9c5f96b6fb7fce5e8cb08ecb6a Mon Sep 17 00:00:00 2001 From: Sasha Levin Date: Sun, 27 Sep 2026 12:16:54 -0400 Subject: [PATCH 102/153] RDMA/selftests: Fix the kernel config fragment The config fragment lists bare option names with no values: CONFIG_TUN CONFIG_VETH CONFIG_DUMMY CONFIG_NET_NS CONFIG_RDMA_RXE which is not valid Kconfig syntax. When the fragment is merged, for example by "make kselftest-merge" or by CI systems, the lines are rejected: .config:5514:warning: unexpected data: CONFIG_TUN .config:5515:warning: unexpected data: CONFIG_VETH .config:5516:warning: unexpected data: CONFIG_DUMMY and none of the options gets enabled, so the rxe tests skip or fail for lack of the modules they load. Give each option a value. The scripts load and unload tun, veth, dummy and rdma_rxe with modprobe, so build those as modules. RDMA_RXE also depends on INFINIBAND, and rxe_rping_between_netns.sh runs rping, which needs the RDMA CM userspace interface provided by INFINIBAND_USER_ACCESS, so add both. Found by KernelCI kselftest builds. Fixes: e01027cab38a ("RDMA/rxe: Add testcase for net namespace rxe") Fixes: ecdcc8a455b7 ("RDMA/selftests: Add rxe_netns_names test") Link: https://patch.msgid.link/r/20260927161655.1184153-1-sashal@kernel.org Assisted-by: LLM Signed-off-by: Sasha Levin Reviewed-by: Zhu Yanjun Signed-off-by: Jason Gunthorpe --- tools/testing/selftests/rdma/config | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/rdma/config b/tools/testing/selftests/rdma/config index e1ff54ec0f57..1be43c4274bb 100644 --- a/tools/testing/selftests/rdma/config +++ b/tools/testing/selftests/rdma/config @@ -1,5 +1,7 @@ -CONFIG_TUN -CONFIG_VETH -CONFIG_DUMMY -CONFIG_NET_NS -CONFIG_RDMA_RXE +CONFIG_TUN=m +CONFIG_VETH=m +CONFIG_DUMMY=m +CONFIG_NET_NS=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_RDMA_RXE=m From a1c9d476c2316685f85fffb6450ad88d01836357 Mon Sep 17 00:00:00 2001 From: Hongyan Xu Date: Mon, 28 Sep 2026 21:41:51 +0800 Subject: [PATCH 103/153] RDMA/hfi1: shut down the trap timer before unregistering MAD processing arms a trap timer embedded in each hfi1_ibport. Device unregistration does not stop the timer, so its callback can run after the port and its containing hfi1_devdata have been released. Shut down every port timer before unregistering the IB device. The shutdown form also prevents a concurrent path from rearming the timer. Fixes: bf90aadd630c ("IB/hfi1: Send MAD traps until repressed") Signed-off-by: Hongyan Xu Link: https://patch.msgid.link/20260928134151.1213-1-getshell@seu.edu.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hfi1/verbs.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index e569b647d611..0ef3b1a89a53 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1888,6 +1888,10 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) void hfi1_unregister_ib_device(struct hfi1_devdata *dd) { struct hfi1_ibdev *dev = &dd->verbs_dev; + int i; + + for (i = 0; i < dd->num_pports; i++) + timer_shutdown_sync(&dd->pport[i].ibport_data.rvp.trap_timer); hfi1_verbs_unregister_sysfs(dd); From e95d076bc1fca7cc36fc8f2f0e9687f79fd314bd Mon Sep 17 00:00:00 2001 From: Hongyan Xu Date: Mon, 28 Sep 2026 21:41:48 +0800 Subject: [PATCH 104/153] RDMA/hfi1: quiesce SDMA asynchronous cleanup on exit The SDMA state reference and completion cover the software cleanup tasklet, but not the hardware cleanup tasklet, err_halt_worker, or flush_worker. Queued instances can therefore access an SDMA engine after per_sdma is released. After the state machine has stopped, cancel err_halt_worker first because it can publish the hardware cleanup tasklet. Then kill that tasklet and drain the remaining flush work. Fixes: 7724105686e7 ("IB/hfi1: add driver files") Signed-off-by: Hongyan Xu Link: https://patch.msgid.link/20260928134148.977-1-getshell@seu.edu.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hfi1/sdma.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c index f253c8ee182d..a71fc8f2d37e 100644 --- a/drivers/infiniband/hw/hfi1/sdma.c +++ b/drivers/infiniband/hw/hfi1/sdma.c @@ -1569,6 +1569,9 @@ void sdma_exit(struct hfi1_devdata *dd) * it is not running. */ sdma_finalput(&sde->state); + cancel_work_sync(&sde->err_halt_worker); + tasklet_kill(&sde->sdma_hw_clean_up_task); + cancel_work_sync(&sde->flush_worker); } } From 9d3259e41ccd5016cd2c6d013232b89df6d73011 Mon Sep 17 00:00:00 2001 From: Runyu Xiao Date: Wed, 30 Sep 2026 15:04:55 +0800 Subject: [PATCH 105/153] IB/hfi1: Shutdown self-rearming timers before freeing The HFI1 receive error and synthetic counter timers rearm themselves from their callbacks. Their cleanup paths use timer_delete_sync(), which only waits for a running callback and does not prevent a concurrent or subsequent rearm. Use timer_shutdown_sync() when releasing these timers. This prevents a callback from enqueueing either timer after cleanup begins, before the containing hfi1_devdata is freed. Preserve the existing work cancellation and workqueue destruction ordering for synthetic counters. Fixes: 7724105686e7 ("IB/hfi1: add driver files") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Runyu Xiao Link: https://patch.msgid.link/20260930070455.2934286-1-runyu.xiao@seu.edu.cn Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hfi1/chip.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 592e330e74bf..faa724ca33bd 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -5567,7 +5567,7 @@ static int init_rcverr(struct hfi1_devdata *dd) static void free_rcverr(struct hfi1_devdata *dd) { if (dd->rcverr_timer.function) - timer_delete_sync(&dd->rcverr_timer); + timer_shutdown_sync(&dd->rcverr_timer); } static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) @@ -12300,7 +12300,7 @@ static void free_cntrs(struct hfi1_devdata *dd) int i; if (dd->synth_stats_timer.function) - timer_delete_sync(&dd->synth_stats_timer); + timer_shutdown_sync(&dd->synth_stats_timer); cancel_work_sync(&dd->update_cntr_work); ppd = (struct hfi1_pportdata *)(dd + 1); for (i = 0; i < dd->num_pports; i++, ppd++) { From 6de34bfb56ad8c24bf63b243b5f32b345d9cdd4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20V=C4=83n=20Cao=20Nguy=C3=AAn?= Date: Wed, 30 Sep 2026 14:51:42 +0700 Subject: [PATCH 106/153] RDMA/irdma: Do not move a fresh RoCE QP to ERROR on RESET MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit irdma_modify_qp_roce() handles IB_QPS_RESET like IB_QPS_ERR and moves the QP to ERROR. On a QP that was never moved to INIT, this makes the following RESET->INIT fail with -EINVAL, while ibv_query_qp() still reports RESET. RESET->RESET is a valid transition. The Mooncake transfer engine resets every new QP before INIT, so it cannot connect over an E810, while mlx5 accepts the same sequence: ibv_create_qp(RC) -> RESET ibv_modify_qp(RESET) -> 0 ibv_modify_qp(INIT, ...) -> -EINVAL Skip the hardware modify when the QP has not been moved to INIT yet (iwarp_state is IRDMA_QP_STATE_INVALID), as there is nothing to flush. RESET after INIT still goes to ERROR. Fixes: b48c24c2d710 ("RDMA/irdma: Implement device supported verb APIs") Assisted-by: LLM Signed-off-by: Nguyá»…n Văn Cao Nguyên Link: https://patch.msgid.link/20260930-irdma-reset-fresh-qp-v1-1-065327274e2b@greennode.ai Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/verbs.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 22a8511fffea..71d7ae7baa47 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -1576,9 +1576,17 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, issue_modify_qp = 1; iwqp->suspend_pending = true; break; + case IB_QPS_RESET: + /* + * A QP that was never moved to INIT has nothing to + * flush, so skip the hardware modify. Moving it to + * ERROR would make the following INIT fail. + */ + if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) + break; + fallthrough; case IB_QPS_SQE: case IB_QPS_ERR: - case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { iwqp->ibqp_state = attr->qp_state; spin_unlock_irqrestore(&iwqp->lock, flags); From 1900af7931ab0cf1ccfb01d3f1a8633482500cde Mon Sep 17 00:00:00 2001 From: Abhijit Gangurde Date: Wed, 30 Sep 2026 14:51:08 +0530 Subject: [PATCH 107/153] RDMA/ionic: support firmware-assigned CQ IDs When the LIF advertises qid allocation for CQ, let firmware return the completion queue ID instead of allocating it in the driver bitmap. Split CQ create/destroy into distinct phases so cq_tbl is updated only after the ID is known and removed before the destroy admin command returns the ID to firmware, avoiding a race where a concurrent create reusing the same cqid could have its xarray entry erased by a stale destroy. Signed-off-by: Abhijit Gangurde Link: https://patch.msgid.link/20260930092111.3176738-3-abhijit.gangurde@amd.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/ionic/ionic_admin.c | 70 +++++++-- .../infiniband/hw/ionic/ionic_controlpath.c | 133 +++++++++++++----- drivers/infiniband/hw/ionic/ionic_fw.h | 15 +- drivers/infiniband/hw/ionic/ionic_ibdev.h | 11 +- drivers/infiniband/hw/ionic/ionic_lif_cfg.c | 1 + drivers/infiniband/hw/ionic/ionic_lif_cfg.h | 1 + 6 files changed, 187 insertions(+), 44 deletions(-) diff --git a/drivers/infiniband/hw/ionic/ionic_admin.c b/drivers/infiniband/hw/ionic/ionic_admin.c index 37e24450d129..93c01bd40d78 100644 --- a/drivers/infiniband/hw/ionic/ionic_admin.c +++ b/drivers/infiniband/hw/ionic/ionic_admin.c @@ -489,6 +489,49 @@ static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev, return ionic_rdma_devcmd(dev, &admin); } +static int ionic_rdma_cq_devcmd(struct ionic_vcq *vcq, + struct ionic_queue *q, + u32 *qid, u32 cid, u16 opcode, + u8 udma_idx) +{ + struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); + struct ionic_admin_ctx admin = { + .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), + .cmd.rdma_queue = { + .opcode = opcode, + .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), + .qid_ver = cpu_to_le32(*qid), + .cid = cpu_to_le32(cid), + .dbid = cpu_to_le16(dev->lif_cfg.dbid), + .depth_log2 = q->depth_log2, + .stride_log2 = q->stride_log2, + .dma_addr = cpu_to_le64(q->dma), + }, + }; + int rc; + + if (ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { + struct ionic_admin_create_cq_resp resp_buf; + + /* Indicates use of CQ create V2 response format. */ + admin.cmd.rdma_queue.qid_ver = cpu_to_le32(IONIC_CREATE_CQ_CMD_V2_MAGIC); + admin.cmd.rdma_queue.udma_idx = udma_idx; + + rc = ionic_rdma_devcmd(dev, &admin); + if (rc) + return rc; + + memcpy(&resp_buf, admin.comp.comp.cmd_data, sizeof(resp_buf)); + *qid = le32_to_cpu(resp_buf.id); + } else { + rc = ionic_rdma_devcmd(dev, &admin); + if (rc) + return rc; + } + + return 0; +} + static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context) { struct ionic_aq *aq = cq_context; @@ -533,12 +576,16 @@ static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev, cq = &vcq->cq[0]; rc = ionic_create_cq_common(vcq, &buf, &attr, NULL, NULL, - NULL, NULL, 0); + NULL, 0); if (rc) goto err_init; - rc = ionic_rdma_queue_devcmd(dev, &cq->q, cq->cqid, cq->eqid, - IONIC_CMD_RDMA_CREATE_CQ); + rc = ionic_rdma_cq_devcmd(vcq, &cq->q, &cq->cqid, cq->eqid, + IONIC_CMD_RDMA_CREATE_CQ, 0); + if (rc) + goto err_cmd; + + rc = ionic_post_create_cq_cmd(cq, NULL, NULL); if (rc) goto err_cmd; @@ -552,6 +599,14 @@ static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev, return ERR_PTR(rc); } +static void ionic_destroy_rdma_admincq(struct ionic_ibdev *dev, + struct ionic_vcq *vcq) +{ + ionic_pre_destroy_cq_cmd(dev, &vcq->cq[0]); + ionic_destroy_cq_common(dev, &vcq->cq[0]); + kfree(vcq); +} + static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev, u32 aqid, u32 cqid) { @@ -1153,8 +1208,7 @@ int ionic_create_rdma_admin(struct ionic_ibdev *dev) vcq->cq[0].cqid); if (IS_ERR(aq)) { /* Clean up the dangling CQ */ - ionic_destroy_cq_common(dev, &vcq->cq[0]); - kfree(vcq); + ionic_destroy_rdma_admincq(dev, vcq); rc = PTR_ERR(aq); @@ -1207,10 +1261,8 @@ void ionic_destroy_rdma_admin(struct ionic_ibdev *dev) cancel_work_sync(&aq->work); __ionic_destroy_rdma_adminq(dev, aq); - if (vcq) { - ionic_destroy_cq_common(dev, &vcq->cq[0]); - kfree(vcq); - } + if (vcq) + ionic_destroy_rdma_admincq(dev, vcq); } kfree(dev->aq_vec); diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index 2b68f377e1fa..4b83657fd384 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -79,12 +79,10 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, struct ionic_ctx *ctx, struct ib_udata *udata, struct ionic_qdesc *req_cq, - __u32 *resp_cqid, int udma_idx) { struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); struct ionic_cq *cq = &vcq->cq[udma_idx]; - void *entry; int rc; cq->vcq = vcq; @@ -94,9 +92,11 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, goto err_args; } - rc = ionic_get_cqid(dev, &cq->cqid, udma_idx); - if (rc) - goto err_args; + if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { + rc = ionic_get_cqid(dev, &cq->cqid, udma_idx); + if (rc) + goto err_args; + } cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx); @@ -123,8 +123,6 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, cq->q.mask = req_cq->mask; cq->q.depth_log2 = req_cq->depth_log2; cq->q.stride_log2 = req_cq->stride_log2; - - *resp_cqid = cq->cqid; } else { rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev, attr->cqe + IONIC_CQ_GRACE, @@ -132,7 +130,6 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, if (rc) goto err_q_init; - ionic_queue_dbell_init(&cq->q, cq->cqid); cq->color = true; cq->credit = cq->q.mask; } @@ -144,20 +141,8 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, init_completion(&cq->cq_rel_comp); kref_init(&cq->cq_kref); - entry = xa_store_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL); - if (entry) { - if (!xa_is_err(entry)) - rc = -EINVAL; - else - rc = xa_err(entry); - - goto err_xa; - } - return 0; -err_xa: - ionic_pgtbl_unbuf(dev, buf); err_pgtbl_init: if (!udata) ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); @@ -165,29 +150,52 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, if (cq->umem) ib_umem_release(cq->umem); err_qdesc: - ionic_put_cqid(dev, cq->cqid); + if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) + ionic_put_cqid(dev, cq->cqid); err_args: cq->vcq = NULL; return rc; } +int ionic_post_create_cq_cmd(struct ionic_cq *cq, + struct ib_udata *udata, + __u32 *resp_cqid) +{ + struct ionic_ibdev *dev = to_ionic_ibdev(cq->vcq->ibcq.device); + + if (udata && resp_cqid) + *resp_cqid = cq->cqid; + else + ionic_queue_dbell_init(&cq->q, cq->cqid); + + return xa_insert_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL); +} + +void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq) +{ + if (!cq->vcq) + return; + + if (!xa_erase_irq(&dev->cq_tbl, cq->cqid)) + return; + + kref_put(&cq->cq_kref, ionic_cq_complete); + wait_for_completion(&cq->cq_rel_comp); +} + void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq) { if (!cq->vcq) return; - xa_erase_irq(&dev->cq_tbl, cq->cqid); - - kref_put(&cq->cq_kref, ionic_cq_complete); - wait_for_completion(&cq->cq_rel_comp); - if (cq->umem) ib_umem_release(cq->umem); else ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); - ionic_put_cqid(dev, cq->cqid); + if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) + ionic_put_cqid(dev, cq->cqid); cq->vcq = NULL; } @@ -1223,8 +1231,11 @@ int ionic_dealloc_mw(struct ib_mw *ibmw) static int ionic_create_cq_cmd(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_cq *cq, - struct ionic_tbl_buf *buf) + struct ionic_tbl_buf *buf, + int udma_idx, + int *out_udma_idx) { + struct ionic_admin_create_cq_resp *resp_buf; const u16 dbid = ionic_ctx_dbid(dev, ctx); struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), @@ -1244,13 +1255,57 @@ static int ionic_create_cq_cmd(struct ionic_ibdev *dev, } } }; + dma_addr_t resp_buf_dma; + int rc; if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ) return -EBADRQC; - ionic_admin_post(dev, &wr); + if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { + ionic_admin_post(dev, &wr); + return ionic_admin_wait(dev, &wr, 0); + } - return ionic_admin_wait(dev, &wr, 0); + resp_buf = kzalloc_obj(*resp_buf); + if (!resp_buf) + return -ENOMEM; + + resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, + sizeof(*resp_buf), + DMA_FROM_DEVICE); + + rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); + if (rc) { + rc = -ENOMEM; + goto err_dma; + } + + wr.wqe.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V2_LEN); + wr.wqe.cmd.create_cq.udma_idx = udma_idx; + wr.wqe.cmd.create_cq.resp_dma_addr = cpu_to_le64(resp_buf_dma); + wr.wqe.cmd.create_cq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN); + + ionic_admin_post(dev, &wr); + rc = ionic_admin_wait(dev, &wr, 0); + + dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), + DMA_FROM_DEVICE); + if (rc) + goto err_dma; + + if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN) { + rc = -EOPNOTSUPP; + goto err_dma; + } + + cq->cqid = le32_to_cpu(resp_buf->id); + if (out_udma_idx) + *out_udma_idx = resp_buf->udma_idx; + +err_dma: + kfree(resp_buf); + + return rc; } static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid) @@ -1309,16 +1364,21 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata, &req.cq[udma_idx], - &resp.cqid[udma_idx], udma_idx); if (rc) goto err_init; - rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf); + rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf, + udma_idx, NULL); if (rc) goto err_cmd; ionic_pgtbl_unbuf(dev, &buf); + + rc = ionic_post_create_cq_cmd(&vcq->cq[udma_idx], udata, + &resp.cqid[udma_idx]); + if (rc) + goto err_post; } vcq->ibcq.cqe = attr->cqe; @@ -1338,6 +1398,8 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, --udma_idx; if (!(vcq->udma_mask & BIT(udma_idx))) continue; + ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]); +err_post: ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); err_cmd: ionic_pgtbl_unbuf(dev, &buf); @@ -1365,6 +1427,13 @@ int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) if (!(vcq->udma_mask & BIT(udma_idx))) continue; + /* + * Untrack the CQ before releasing its hardware ID below, so a + * concurrent create that gets the same ID reused by firmware + * cannot have its fresh XArray entry erased by this destroy. + */ + ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]); + rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); if (rc_tmp) { if (!rc) diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h index d0a5bce85ed8..b299a1fe5acb 100644 --- a/drivers/infiniband/hw/ionic/ionic_fw.h +++ b/drivers/infiniband/hw/ionic/ionic_fw.h @@ -746,11 +746,22 @@ struct ionic_admin_create_cq { __le32 map_count; __le64 dma_addr; __le16 dbid_flags; + __u8 udma_idx; + __le32 resp_buf_len; + __le64 resp_dma_addr; } __packed; #define IONIC_ADMIN_CREATE_CQ_IN_V1_LEN 34 -static_assert(sizeof(struct ionic_admin_create_cq) == - IONIC_ADMIN_CREATE_CQ_IN_V1_LEN); +#define IONIC_ADMIN_CREATE_CQ_IN_V2_LEN 47 +static_assert(sizeof(struct ionic_admin_create_cq) == IONIC_ADMIN_CREATE_CQ_IN_V2_LEN); + +struct ionic_admin_create_cq_resp { + __le32 id; + __u8 udma_idx; +} __packed; + +#define IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN 5 +static_assert(sizeof(struct ionic_admin_create_cq_resp) == IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN); struct ionic_admin_destroy_cq { __le32 cq_id; diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h index 32b6a8a45fa2..11374dd5a309 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.h +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -438,6 +438,12 @@ static inline void ionic_cq_complete(struct kref *kref) complete(&cq->cq_rel_comp); } +static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev, + enum ionic_lif_rdma_alloc_qid qtype) +{ + return dev->lif_cfg.alloc_qid_cap & qtype; +} + /* ionic_admin.c */ extern struct workqueue_struct *ionic_evt_workq; void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); @@ -457,8 +463,11 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, struct ionic_ctx *ctx, struct ib_udata *udata, struct ionic_qdesc *req_cq, - __u32 *resp_cqid, int udma_idx); +int ionic_post_create_cq_cmd(struct ionic_cq *cq, + struct ib_udata *udata, + __u32 *resp_cqid); +void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq); void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq); void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp); void ionic_notify_flush_cq(struct ionic_cq *cq); diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c index 1e37bd09490f..c381bda4360d 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c @@ -93,6 +93,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) !!(lif->qtype_info[IONIC_QTYPE_TXQ].features & IONIC_QIDENT_F_EXPDB); cfg->rq_expdb = !!(lif->qtype_info[IONIC_QTYPE_RXQ].features & IONIC_QIDENT_F_EXPDB); + cfg->alloc_qid_cap = ident->rdma.alloc_qid_cap; } struct net_device *ionic_lif_netdev(struct ionic_lif *lif) diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h index d7835ac27896..68aec06908ca 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h @@ -58,6 +58,7 @@ struct ionic_lif_cfg { bool rq_expdb; u8 expdb_mask; u8 rcq_sign_bit; + u8 alloc_qid_cap; }; void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg); From 0d0c461f814b04f5f9cea007fd276784b7ed7c17 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 30 Sep 2026 14:51:09 +0530 Subject: [PATCH 108/153] RDMA/ionic: segregate rq related fields from ionic_qp into a new ionic_rq struct In preparation for SRQ support, segregate rq related fields from ionic_qp into a new ionic_rq struct. This refactoring enables both non-SRQ and future SRQ code to leverage the common rq init/destroy infrastructure around ionic_qp_rq_init/_destroy with no functional changes. Signed-off-by: Amit Kumar Signed-off-by: Abhijit Gangurde Link: https://patch.msgid.link/20260930092111.3176738-4-abhijit.gangurde@amd.com Signed-off-by: Leon Romanovsky --- .../infiniband/hw/ionic/ionic_controlpath.c | 301 +++++++++--------- drivers/infiniband/hw/ionic/ionic_datapath.c | 90 +++--- drivers/infiniband/hw/ionic/ionic_ibdev.h | 40 ++- 3 files changed, 218 insertions(+), 213 deletions(-) diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index 4b83657fd384..f77a49f4231a 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -1466,8 +1466,8 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject); const u32 flags = to_ionic_qp_flags(0, 0, qp->sq_cmb & IONIC_CMB_ENABLE, - qp->rq_cmb & IONIC_CMB_ENABLE, - qp->sq_spec, qp->rq_spec, + qp->rq.cmb & IONIC_CMB_ENABLE, + qp->sq_spec, qp->rq.spec, pd->flags & IONIC_QPF_PRIVILEGED, pd_remote_privileged(&pd->ibpd)); struct ionic_admin_wr wr = { @@ -1506,8 +1506,8 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, if (qp->has_rq) { wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); - wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.depth_log2; - wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.stride_log2; + wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.q.depth_log2; + wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.q.stride_log2; wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2; wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0); wr.wqe.cmd.create_qp.rq_map_count = @@ -1529,8 +1529,8 @@ static int ionic_modify_qp_cmd(struct ionic_ibdev *dev, const u32 flags = to_ionic_qp_flags(attr->qp_access_flags, attr->en_sqd_async_notify, qp->sq_cmb & IONIC_CMB_ENABLE, - qp->rq_cmb & IONIC_CMB_ENABLE, - qp->sq_spec, qp->rq_spec, + qp->rq.cmb & IONIC_CMB_ENABLE, + qp->sq_spec, qp->rq.spec, pd->flags & IONIC_QPF_PRIVILEGED, pd_remote_privileged(qp->ibqp.pd)); const u8 state = to_ionic_qp_modify_state(attr->qp_state, @@ -1700,9 +1700,9 @@ static int ionic_query_qp_cmd(struct ionic_ibdev *dev, if (qp->has_rq) { attr->cap.max_recv_sge = - ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, - qp->rq_spec, - qp->rq_cmb & IONIC_CMB_EXPDB); + ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, + qp->rq.spec, + qp->rq.cmb & IONIC_CMB_EXPDB); } query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); @@ -2072,99 +2072,87 @@ static void ionic_qp_sq_destroy(struct ionic_ibdev *dev, ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); } -static void ionic_qp_rq_init_cmb(struct ionic_ibdev *dev, - struct ionic_qp *qp, - struct ib_udata *udata) +static void ionic_rq_init_cmb(struct ionic_ibdev *dev, + struct ionic_rq *rq, + struct ib_udata *udata) { u8 expdb_stride_log2 = 0; bool expdb; int rc; - if (!(qp->rq_cmb & IONIC_CMB_ENABLE)) + if (!(rq->cmb & IONIC_CMB_ENABLE)) goto not_in_cmb; - if (qp->rq_cmb & ~IONIC_CMB_SUPPORTED) { - if (qp->rq_cmb & IONIC_CMB_REQUIRE) + if (rq->cmb & ~IONIC_CMB_SUPPORTED) { + if (rq->cmb & IONIC_CMB_REQUIRE) goto not_in_cmb; - qp->rq_cmb &= IONIC_CMB_SUPPORTED; + rq->cmb &= IONIC_CMB_SUPPORTED; } - if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) { - if (qp->rq_cmb & IONIC_CMB_REQUIRE) + if ((rq->cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) { + if (rq->cmb & IONIC_CMB_REQUIRE) goto not_in_cmb; - qp->rq_cmb &= ~IONIC_CMB_EXPDB; + rq->cmb &= ~IONIC_CMB_EXPDB; } - qp->rq_cmb_order = order_base_2(qp->rq.size / PAGE_SIZE); + rq->cmb_order = order_base_2(rq->q.size / PAGE_SIZE); - if (qp->rq_cmb_order >= IONIC_RQCMB_ORDER) + if (rq->cmb_order >= IONIC_RQCMB_ORDER) goto not_in_cmb; - if (qp->rq_cmb & IONIC_CMB_EXPDB) - expdb_stride_log2 = qp->rq.stride_log2; + if (rq->cmb & IONIC_CMB_EXPDB) + expdb_stride_log2 = rq->q.stride_log2; - rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->rq_cmb_pgid, - &qp->rq_cmb_addr, qp->rq_cmb_order, + rc = ionic_get_cmb(dev->lif_cfg.lif, &rq->cmb_pgid, + &rq->cmb_addr, rq->cmb_order, expdb_stride_log2, &expdb); if (rc) goto not_in_cmb; - if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !expdb) { - if (qp->rq_cmb & IONIC_CMB_REQUIRE) + if ((rq->cmb & IONIC_CMB_EXPDB) && !expdb) { + if (rq->cmb & IONIC_CMB_REQUIRE) goto err_map; - qp->rq_cmb &= ~IONIC_CMB_EXPDB; + rq->cmb &= ~IONIC_CMB_EXPDB; } return; err_map: - ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order); + ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order); not_in_cmb: - if (qp->rq_cmb & IONIC_CMB_REQUIRE) + if (rq->cmb & IONIC_CMB_REQUIRE) ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n"); - qp->rq_cmb = 0; - qp->rq_cmb_order = IONIC_RES_INVALID; - qp->rq_cmb_pgid = 0; - qp->rq_cmb_addr = 0; + rq->cmb = 0; + rq->cmb_order = IONIC_RES_INVALID; + rq->cmb_pgid = 0; + rq->cmb_addr = 0; } -static void ionic_qp_rq_destroy_cmb(struct ionic_ibdev *dev, - struct ionic_ctx *ctx, - struct ionic_qp *qp) +static void ionic_rq_destroy_cmb(struct ionic_ibdev *dev, + struct ionic_ctx *ctx, + struct ionic_rq *rq) { - if (!(qp->rq_cmb & IONIC_CMB_ENABLE)) + if (!(rq->cmb & IONIC_CMB_ENABLE)) return; if (ctx) - rdma_user_mmap_entry_remove(qp->mmap_rq_cmb); + rdma_user_mmap_entry_remove(rq->mmap_cmb); - ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order); + ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order); } -static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, - struct ionic_qp *qp, struct ionic_qdesc *rq, - struct ionic_tbl_buf *buf, int max_wr, int max_sge, - int rq_spec, struct ib_udata *udata) +static int ionic_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, + struct ionic_rq *rq, struct ionic_qdesc *qdesc, + struct ionic_tbl_buf *buf, int max_wr, int max_sge, + int rq_spec, struct ib_udata *udata) { int rc = 0, i; u32 wqe_size; - if (!qp->has_rq) { - if (buf) { - buf->tbl_buf = NULL; - buf->tbl_limit = 0; - buf->tbl_pages = 0; - } - if (udata) - rc = ionic_validate_qdesc_zero(rq); - - return rc; - } - rc = -EINVAL; if (max_wr < 0 || max_wr > 0xffff) @@ -2178,105 +2166,102 @@ static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, return rc; if (udata) { - rc = ionic_validate_qdesc(rq); + rc = ionic_validate_qdesc(qdesc); if (rc) return rc; - qp->rq_spec = rq_spec; + rq->spec = rq_spec; - qp->rq.ptr = NULL; - qp->rq.size = rq->size; - qp->rq.mask = rq->mask; - qp->rq.depth_log2 = rq->depth_log2; - qp->rq.stride_log2 = rq->stride_log2; + rq->q.ptr = NULL; + rq->q.size = qdesc->size; + rq->q.mask = qdesc->mask; + rq->q.depth_log2 = qdesc->depth_log2; + rq->q.stride_log2 = qdesc->stride_log2; - qp->rq_meta = NULL; + rq->meta = NULL; - qp->rq_umem = ib_umem_get_va(&dev->ibdev, rq->addr, rq->size, 0); - if (IS_ERR(qp->rq_umem)) - return PTR_ERR(qp->rq_umem); + rq->umem = ib_umem_get_va(&dev->ibdev, qdesc->addr, qdesc->size, 0); + if (IS_ERR(rq->umem)) + return PTR_ERR(rq->umem); } else { - qp->rq_umem = NULL; + rq->umem = NULL; - qp->rq_spec = ionic_v1_use_spec_sge(max_sge, rq_spec); - if (rq_spec && !qp->rq_spec) + rq->spec = ionic_v1_use_spec_sge(max_sge, rq_spec); + if (rq_spec && !rq->spec) ibdev_dbg(&dev->ibdev, "init rq: max_sge %u disables spec\n", max_sge); - if (qp->rq_cmb & IONIC_CMB_EXPDB) { + if (rq->cmb & IONIC_CMB_EXPDB) { wqe_size = ionic_v1_recv_wqe_min_size(max_sge, - qp->rq_spec, + rq->spec, true); if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) - qp->rq_cmb &= ~IONIC_CMB_EXPDB; + rq->cmb &= ~IONIC_CMB_EXPDB; } - if (!(qp->rq_cmb & IONIC_CMB_EXPDB)) + if (!(rq->cmb & IONIC_CMB_EXPDB)) wqe_size = ionic_v1_recv_wqe_min_size(max_sge, - qp->rq_spec, + rq->spec, false); - rc = ionic_queue_init(&qp->rq, dev->lif_cfg.hwdev, + rc = ionic_queue_init(&rq->q, dev->lif_cfg.hwdev, max_wr, wqe_size); if (rc) return rc; - ionic_queue_dbell_init(&qp->rq, qp->qpid); + ionic_queue_dbell_init(&rq->q, rq->qid); - qp->rq_meta = kmalloc_objs(*qp->rq_meta, (u32)qp->rq.mask + 1); - if (!qp->rq_meta) { + rq->meta = kmalloc_objs(*rq->meta, (u32)rq->q.mask + 1); + if (!rq->meta) { rc = -ENOMEM; goto err_rq_meta; } - for (i = 0; i < qp->rq.mask; ++i) - qp->rq_meta[i].next = &qp->rq_meta[i + 1]; - qp->rq_meta[i].next = IONIC_META_LAST; - qp->rq_meta_head = &qp->rq_meta[0]; + for (i = 0; i < rq->q.mask; ++i) + rq->meta[i].next = &rq->meta[i + 1]; + rq->meta[i].next = IONIC_META_LAST; + rq->meta_head = &rq->meta[0]; } - ionic_qp_rq_init_cmb(dev, qp, udata); + ionic_rq_init_cmb(dev, rq, udata); - if (qp->rq_cmb & IONIC_CMB_ENABLE) + if (rq->cmb & IONIC_CMB_ENABLE) rc = ionic_pgtbl_init(dev, buf, NULL, - (u64)qp->rq_cmb_pgid << PAGE_SHIFT, + (u64)rq->cmb_pgid << PAGE_SHIFT, 1, PAGE_SIZE); else - rc = ionic_pgtbl_init(dev, buf, - qp->rq_umem, qp->rq.dma, 1, PAGE_SIZE); + rc = ionic_pgtbl_init(dev, buf, rq->umem, + rq->q.dma, 1, PAGE_SIZE); if (rc) goto err_rq_tbl; return 0; err_rq_tbl: - ionic_qp_rq_destroy_cmb(dev, ctx, qp); - kfree(qp->rq_meta); + ionic_rq_destroy_cmb(dev, ctx, rq); + kfree(rq->meta); err_rq_meta: - if (qp->rq_umem) - ib_umem_release(qp->rq_umem); + if (rq->umem) + ib_umem_release(rq->umem); else - ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev); + ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); return rc; } -static void ionic_qp_rq_destroy(struct ionic_ibdev *dev, - struct ionic_ctx *ctx, - struct ionic_qp *qp) +static void ionic_rq_destroy(struct ionic_ibdev *dev, + struct ionic_ctx *ctx, + struct ionic_rq *rq) { - if (!qp->has_rq) - return; + ionic_rq_destroy_cmb(dev, ctx, rq); - ionic_qp_rq_destroy_cmb(dev, ctx, qp); + kfree(rq->meta); - kfree(qp->rq_meta); - - if (qp->rq_umem) - ib_umem_release(qp->rq_umem); + if (rq->umem) + ib_umem_release(rq->umem); else - ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev); + ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); } int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, @@ -2317,10 +2302,10 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, INIT_LIST_HEAD(&qp->cq_poll_sq); INIT_LIST_HEAD(&qp->cq_flush_sq); - INIT_LIST_HEAD(&qp->cq_flush_rq); + INIT_LIST_HEAD(&qp->rq.cq_flush); spin_lock_init(&qp->sq_lock); - spin_lock_init(&qp->rq_lock); + spin_lock_init(&qp->rq.lock); qp->has_sq = 1; qp->has_rq = 1; @@ -2364,7 +2349,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, if (udata) { if (req.rq_cmb & IONIC_CMB_ENABLE) - qp->rq_cmb = req.rq_cmb; + qp->rq.cmb = req.rq_cmb; if (req.sq_cmb & IONIC_CMB_ENABLE) qp->sq_cmb = req.sq_cmb; @@ -2376,11 +2361,22 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, if (rc) goto err_sq; - rc = ionic_qp_rq_init(dev, ctx, qp, &req.rq, &rq_buf, - attr->cap.max_recv_wr, attr->cap.max_recv_sge, - req.rq_spec, udata); - if (rc) - goto err_rq; + if (qp->has_rq) { + /* for non-srq qps, rq qid is same as qpid */ + qp->rq.qid = qp->qpid; + rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf, + attr->cap.max_recv_wr, attr->cap.max_recv_sge, + req.rq_spec, udata); + if (rc) + goto err_rq; + } else { + rq_buf.tbl_buf = NULL; + rq_buf.tbl_limit = 0; + rq_buf.tbl_pages = 0; + + if (udata) + rc = ionic_validate_qdesc_zero(&req.rq); + } rc = ionic_create_qp_cmd(dev, pd, to_ionic_vcq_cq(attr->send_cq, qp->udma_idx), @@ -2426,41 +2422,41 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, resp.sq_cmb = qp->sq_cmb; } - if (qp->rq_cmb & IONIC_CMB_ENABLE) { + if (qp->rq.cmb & IONIC_CMB_ENABLE) { bool wc; - if ((qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == + if ((qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == (IONIC_CMB_WC | IONIC_CMB_UC)) { ibdev_dbg(&dev->ibdev, - "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); - qp->rq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); + "Both rq.cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); + qp->rq.cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); } - if (qp->rq_cmb & IONIC_CMB_EXPDB) - wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) + if (qp->rq.cmb & IONIC_CMB_EXPDB) + wc = (qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == IONIC_CMB_WC; else - wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) + wc = (qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) != IONIC_CMB_UC; /* let userspace know the mapping */ if (wc) - qp->rq_cmb |= IONIC_CMB_WC; + qp->rq.cmb |= IONIC_CMB_WC; else - qp->rq_cmb |= IONIC_CMB_UC; + qp->rq.cmb |= IONIC_CMB_UC; - qp->mmap_rq_cmb = + qp->rq.mmap_cmb = ionic_mmap_entry_insert(ctx, - qp->rq.size, - PHYS_PFN(qp->rq_cmb_addr), + qp->rq.q.size, + PHYS_PFN(qp->rq.cmb_addr), wc ? IONIC_MMAP_WC : 0, &resp.rq_cmb_offset); - if (!qp->mmap_rq_cmb) { + if (!qp->rq.mmap_cmb) { rc = -ENOMEM; goto err_qp_cmd; } - resp.rq_cmb = qp->rq_cmb; + resp.rq_cmb = qp->rq.cmb; } rc = ib_respond_udata(udata, resp); @@ -2504,12 +2500,12 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, if (qp->has_rq) { cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx); - attr->cap.max_recv_wr = qp->rq.mask; + attr->cap.max_recv_wr = qp->rq.q.mask; attr->cap.max_recv_sge = - ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, - qp->rq_spec, - qp->rq_cmb & IONIC_CMB_EXPDB); - qp->rq_cqid = cq->cqid; + ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, + qp->rq.spec, + qp->rq.cmb & IONIC_CMB_EXPDB); + qp->rq.cqid = cq->cqid; } return 0; @@ -2518,7 +2514,8 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, ionic_destroy_qp_cmd(dev, qp->qpid); err_cmd: ionic_pgtbl_unbuf(dev, &rq_buf); - ionic_qp_rq_destroy(dev, ctx, qp); + if (qp->has_rq) + ionic_rq_destroy(dev, ctx, &qp->rq); err_rq: ionic_pgtbl_unbuf(dev, &sq_buf); ionic_qp_sq_destroy(dev, ctx, qp); @@ -2572,15 +2569,15 @@ void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) if (qp->ibqp.recv_cq) { cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); - /* Hold the CQ lock and QP rq_lock to set up flush */ + /* Hold the CQ lock and QP rq.lock to set up flush */ spin_lock_irqsave(&cq->lock, irqflags); - spin_lock(&qp->rq_lock); - qp->rq_flush = true; - if (!ionic_queue_empty(&qp->rq)) { + spin_lock(&qp->rq.lock); + qp->rq.flush = true; + if (!ionic_queue_empty(&qp->rq.q)) { cq->flush = true; - list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); + list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); } - spin_unlock(&qp->rq_lock); + spin_unlock(&qp->rq.lock); spin_unlock_irqrestore(&cq->lock, irqflags); } } @@ -2646,17 +2643,17 @@ static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) } if (qp->has_rq) { - spin_lock(&qp->rq_lock); - qp->rq_flush = false; - qp->rq.prod = 0; - qp->rq.cons = 0; - if (qp->rq_meta) { - for (i = 0; i < qp->rq.mask; ++i) - qp->rq_meta[i].next = &qp->rq_meta[i + 1]; - qp->rq_meta[i].next = IONIC_META_LAST; + spin_lock(&qp->rq.lock); + qp->rq.flush = false; + qp->rq.q.prod = 0; + qp->rq.q.cons = 0; + if (qp->rq.meta) { + for (i = 0; i < qp->rq.q.mask; ++i) + qp->rq.meta[i].next = &qp->rq.meta[i + 1]; + qp->rq.meta[i].next = IONIC_META_LAST; } - qp->rq_meta_head = &qp->rq_meta[0]; - spin_unlock(&qp->rq_lock); + qp->rq.meta_head = &qp->rq.meta[0]; + spin_unlock(&qp->rq.lock); } local_irq_restore(irqflags); @@ -2774,7 +2771,7 @@ int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, attr->cap.max_send_wr = qp->sq.mask; if (qp->has_rq) - attr->cap.max_recv_wr = qp->rq.mask; + attr->cap.max_recv_wr = qp->rq.q.mask; init_attr->event_handler = ibqp->event_handler; init_attr->qp_context = ibqp->qp_context; @@ -2830,11 +2827,13 @@ int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); spin_lock_irqsave(&cq->lock, irqflags); ionic_clean_cq(cq, qp->qpid); - list_del(&qp->cq_flush_rq); + list_del(&qp->rq.cq_flush); spin_unlock_irqrestore(&cq->lock, irqflags); } - ionic_qp_rq_destroy(dev, ctx, qp); + if (qp->has_rq) + ionic_rq_destroy(dev, ctx, &qp->rq); + ionic_qp_sq_destroy(dev, ctx, qp); if (qp->has_ah) { ionic_put_ahid(dev, qp->ahid); diff --git a/drivers/infiniband/hw/ionic/ionic_datapath.c b/drivers/infiniband/hw/ionic/ionic_datapath.c index 3e2300f7ea10..522c6f08c9fd 100644 --- a/drivers/infiniband/hw/ionic/ionic_datapath.c +++ b/drivers/infiniband/hw/ionic/ionic_datapath.c @@ -34,17 +34,17 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) struct ionic_v1_wqe *wqe; u64 wqe_idx; - if (!qp->rq_flush) + if (!qp->rq.flush) return 0; - if (ionic_queue_empty(&qp->rq)) + if (ionic_queue_empty(&qp->rq.q)) return 0; - wqe = ionic_queue_at_cons(&qp->rq); + wqe = ionic_queue_at_cons(&qp->rq.q); wqe_idx = le64_to_cpu(wqe->base.wqe_idx); /* wqe_idx must be a valid queue index */ - if (unlikely(wqe_idx >> qp->rq.depth_log2)) { + if (unlikely(wqe_idx >> qp->rq.q.depth_log2)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu invalid\n", qp->qpid, (unsigned long long)wqe_idx); @@ -52,7 +52,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) } /* wqe_idx must indicate a request that is outstanding */ - meta = &qp->rq_meta[wqe_idx]; + meta = &qp->rq.meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu not posted\n", @@ -60,7 +60,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) return -EIO; } - ionic_queue_consume(&qp->rq); + ionic_queue_consume(&qp->rq.q); memset(wc, 0, sizeof(*wc)); @@ -68,8 +68,8 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) wc->wr_id = meta->wrid; wc->qp = &qp->ibqp; - meta->next = qp->rq_meta_head; - qp->rq_meta_head = meta; + meta->next = qp->rq.meta_head; + qp->rq.meta_head = meta; return 1; } @@ -139,7 +139,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, u32 src_qpn, st_len; u8 op; - if (cqe_qp->rq_flush) + if (cqe_qp->rq.flush) return 0; qp = cqe_qp; @@ -148,38 +148,38 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, /* ignore wqe_idx in case of flush error */ if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { - cqe_qp->rq_flush = true; + cqe_qp->rq.flush = true; cq->flush = true; - list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); + list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); /* posted recvs (if any) flushed by ionic_flush_recv */ return 0; } /* there had better be something in the recv queue to complete */ - if (ionic_queue_empty(&qp->rq)) { + if (ionic_queue_empty(&qp->rq.q)) { ibdev_warn(&dev->ibdev, "qp %u is empty\n", qp->qpid); return -EIO; } wqe_idx = le64_to_cpu(cqe->recv.wqe_idx_timestamp) & IONIC_V1_CQE_WQE_IDX_MASK; /* wqe_idx must be a valid queue index */ - if (unlikely(wqe_idx >> qp->rq.depth_log2)) { + if (unlikely(wqe_idx >> qp->rq.q.depth_log2)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u invalid\n", qp->qpid, wqe_idx); return -EIO; } /* wqe_idx must indicate a request that is outstanding */ - meta = &qp->rq_meta[wqe_idx]; + meta = &qp->rq.meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u not posted\n", qp->qpid, wqe_idx); return -EIO; } - meta->next = qp->rq_meta_head; - qp->rq_meta_head = meta; + meta->next = qp->rq.meta_head; + qp->rq.meta_head = meta; memset(wc, 0, sizeof(*wc)); @@ -191,9 +191,9 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, wc->vendor_err = st_len; wc->status = ionic_to_ib_status(st_len); - cqe_qp->rq_flush = true; + cqe_qp->rq.flush = true; cq->flush = true; - list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); + list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); ibdev_warn(&dev->ibdev, "qp %d recv cqe with error\n", qp->qpid); @@ -255,7 +255,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, wc->port_num = 1; out: - ionic_queue_consume(&qp->rq); + ionic_queue_consume(&qp->rq.q); return 1; } @@ -526,9 +526,9 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev, switch (type) { case IONIC_V1_CQE_TYPE_RECV: - spin_lock(&qp->rq_lock); + spin_lock(&qp->rq.lock); rc = ionic_poll_recv(dev, cq, qp, cqe, wc + npolled); - spin_unlock(&qp->rq_lock); + spin_unlock(&qp->rq.lock); if (rc < 0) goto out; @@ -612,19 +612,19 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev, cq->flush = true; } - list_for_each_entry_safe(qp, qp_next, &cq->flush_rq, cq_flush_rq) { + list_for_each_entry_safe(qp, qp_next, &cq->flush_rq, rq.cq_flush) { if (npolled == nwc) goto out; - spin_lock(&qp->rq_lock); + spin_lock(&qp->rq.lock); rc = ionic_flush_recv_many(qp, wc + npolled, nwc - npolled); - spin_unlock(&qp->rq_lock); + spin_unlock(&qp->rq.lock); if (rc > 0) npolled += rc; if (npolled < nwc) - list_del_init(&qp->cq_flush_rq); + list_del_init(&qp->rq.cq_flush); else cq->flush = true; } @@ -881,7 +881,7 @@ static void ionic_prep_sq_wqe(struct ionic_qp *qp, void *wqe) static void ionic_prep_rq_wqe(struct ionic_qp *qp, void *wqe) { - memset(wqe, 0, 1u << qp->rq.stride_log2); + memset(wqe, 0, 1u << qp->rq.q.stride_log2); } static int ionic_prep_send(struct ionic_qp *qp, @@ -1183,38 +1183,38 @@ static int ionic_prep_recv(struct ionic_qp *qp, s64 signed_len; u32 mval; - wqe = ionic_queue_at_prod(&qp->rq); + wqe = ionic_queue_at_prod(&qp->rq.q); /* if wqe is owned by device, caller can try posting again soon */ if (wqe->base.flags & cpu_to_be16(IONIC_V1_FLAG_FENCE)) return -EAGAIN; - meta = qp->rq_meta_head; + meta = qp->rq.meta_head; if (unlikely(meta == IONIC_META_LAST) || unlikely(meta == IONIC_META_POSTED)) return -EIO; ionic_prep_rq_wqe(qp, wqe); - mval = ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, qp->rq_spec, + mval = ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, qp->rq.spec, false); signed_len = ionic_prep_pld(wqe, &wqe->recv.pld, - qp->rq_spec, mval, + qp->rq.spec, mval, wr->sg_list, wr->num_sge); if (signed_len < 0) return signed_len; meta->wrid = wr->wr_id; - wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq_meta); + wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq.meta); wqe->base.num_sge_key = wr->num_sge; /* total length for recv goes in base imm_data_key */ wqe->base.imm_data_key = cpu_to_be32(signed_len); - ionic_queue_produce(&qp->rq); + ionic_queue_produce(&qp->rq.q); - qp->rq_meta_head = meta->next; + qp->rq.meta_head = meta->next; meta->next = IONIC_META_POSTED; return 0; @@ -1321,10 +1321,10 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev, return -EINVAL; } - spin_lock_irqsave(&qp->rq_lock, irqflags); + spin_lock_irqsave(&qp->rq.lock, irqflags); while (wr) { - if (ionic_queue_full(&qp->rq)) { + if (ionic_queue_full(&qp->rq.q)) { ibdev_dbg(&dev->ibdev, "queue full"); rc = -ENOMEM; goto out; @@ -1339,32 +1339,32 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev, out: if (!cq) { - spin_unlock_irqrestore(&qp->rq_lock, irqflags); + spin_unlock_irqrestore(&qp->rq.lock, irqflags); goto out_unlocked; } - spin_unlock_irqrestore(&qp->rq_lock, irqflags); + spin_unlock_irqrestore(&qp->rq.lock, irqflags); spin_lock_irqsave(&cq->lock, irqflags); - spin_lock(&qp->rq_lock); + spin_lock(&qp->rq.lock); - if (likely(qp->rq.prod != qp->rq_old_prod)) { + if (likely(qp->rq.q.prod != qp->rq.old_prod)) { /* ring cq doorbell just in time */ - spend = (qp->rq.prod - qp->rq_old_prod) & qp->rq.mask; + spend = (qp->rq.q.prod - qp->rq.old_prod) & qp->rq.q.mask; ionic_reserve_cq(dev, cq, spend); - qp->rq_old_prod = qp->rq.prod; + qp->rq.old_prod = qp->rq.q.prod; ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.rq_qtype, - ionic_queue_dbell_val(&qp->rq)); + ionic_queue_dbell_val(&qp->rq.q)); } - if (qp->rq_flush) { + if (qp->rq.flush) { notify = true; cq->flush = true; - list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); + list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); } - spin_unlock(&qp->rq_lock); + spin_unlock(&qp->rq.lock); spin_unlock_irqrestore(&cq->lock, irqflags); if (notify && vcq->ibcq.comp_handler) diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h index 11374dd5a309..aa3b4122c5ec 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.h +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -241,6 +241,28 @@ struct ionic_rq_meta { u64 wrid; }; +struct ionic_rq { + struct ionic_queue q; + u32 qid; + u32 cqid; + struct list_head cq_flush; + spinlock_t lock; /* for posting and polling */ + + phys_addr_t cmb_addr; + struct rdma_user_mmap_entry *mmap_cmb; + + struct ionic_rq_meta *meta; + struct ionic_rq_meta *meta_head; + struct ib_umem *umem; + + int spec; + int cmb_order; + u32 cmb_pgid; + u16 old_prod; + u8 cmb; + bool flush; +}; + struct ionic_qp { struct ib_qp ibqp; enum ib_qp_state state; @@ -248,7 +270,6 @@ struct ionic_qp { u32 qpid; u32 ahid; u32 sq_cqid; - u32 rq_cqid; u8 udma_idx; u8 has_ah:1; u8 has_sq:1; @@ -259,7 +280,6 @@ struct ionic_qp { struct list_head cq_poll_sq; struct list_head cq_flush_sq; - struct list_head cq_flush_rq; struct list_head ibkill_flush_ent; spinlock_t sq_lock; /* for posting and polling */ @@ -274,14 +294,7 @@ struct ionic_qp { bool sq_flush; bool sq_flush_rcvd; - spinlock_t rq_lock; /* for posting and polling */ - struct ionic_queue rq; - struct ionic_rq_meta *rq_meta; - struct ionic_rq_meta *rq_meta_head; - int rq_spec; - u16 rq_old_prod; - u8 rq_cmb; - bool rq_flush; + struct ionic_rq rq; struct kref qp_kref; struct completion qp_rel_comp; @@ -295,13 +308,6 @@ struct ionic_qp { struct ib_umem *sq_umem; - int rq_cmb_order; - u32 rq_cmb_pgid; - phys_addr_t rq_cmb_addr; - struct rdma_user_mmap_entry *mmap_rq_cmb; - - struct ib_umem *rq_umem; - int dcqcn_profile; struct ib_ud_header *hdr; From 8a093abb541a8f57e60bea2b9325ca9db63d6f83 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 30 Sep 2026 14:51:10 +0530 Subject: [PATCH 109/153] RDMA/ionic: add Shared receive queue (SRQ) support Implement device supported verb APIs for shared receive queue. SRQ is supported for userspace only. ionic_create_srq() returns -EOPNOTSUPP when there is no ucontext, so kernel ULPs cannot create an SRQ. Signed-off-by: Amit Kumar Signed-off-by: Abhijit Gangurde Link: https://patch.msgid.link/20260930092111.3176738-5-abhijit.gangurde@amd.com Signed-off-by: Leon Romanovsky --- .../infiniband/hw/ionic/ionic_controlpath.c | 395 +++++++++++++++--- drivers/infiniband/hw/ionic/ionic_datapath.c | 8 +- drivers/infiniband/hw/ionic/ionic_fw.h | 53 +++ drivers/infiniband/hw/ionic/ionic_ibdev.c | 16 + drivers/infiniband/hw/ionic/ionic_ibdev.h | 24 +- drivers/infiniband/hw/ionic/ionic_lif_cfg.c | 1 + drivers/infiniband/hw/ionic/ionic_lif_cfg.h | 1 + include/uapi/rdma/ionic-abi.h | 2 +- 8 files changed, 441 insertions(+), 59 deletions(-) diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index f77a49f4231a..3e7354361e40 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -1504,7 +1504,7 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0); } - if (qp->has_rq) { + if (!qp->srq) { wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.q.depth_log2; wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.q.stride_log2; @@ -1513,6 +1513,9 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, wr.wqe.cmd.create_qp.rq_map_count = cpu_to_le32(rq_buf->tbl_pages); wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0); + } else { + wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(qp->srq->rq.qid); + wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); } ionic_admin_post(dev, &wr); @@ -1698,7 +1701,7 @@ static int ionic_query_qp_cmd(struct ionic_ibdev *dev, ionic_v1_send_wqe_max_data(qp->sq.stride_log2, expdb); } - if (qp->has_rq) { + if (!qp->srq) { attr->cap.max_recv_sge = ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, qp->rq.spec, @@ -2072,6 +2075,42 @@ static void ionic_qp_sq_destroy(struct ionic_ibdev *dev, ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); } +static void ionic_rq_mmap_cmb(struct ionic_ibdev *dev, + struct ionic_ctx *ctx, + struct ionic_rq *rq, + u64 *cmb_offset) +{ + bool wc; + + /* set mapping by default to uncached for + * expdb (to guarantee writes order) otherwise + * writecombine, unless this default is + * overridden by userspace + */ + if ((rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == + (IONIC_CMB_WC | IONIC_CMB_UC)) { + ibdev_dbg(&dev->ibdev, + "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC set, using default driver mapping\n"); + rq->cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); + } + + if (rq->cmb & IONIC_CMB_EXPDB) + wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == IONIC_CMB_WC; + else + wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) != IONIC_CMB_UC; + + /* let userspace know the mapping */ + if (wc) + rq->cmb |= IONIC_CMB_WC; + else + rq->cmb |= IONIC_CMB_UC; + + rq->mmap_cmb = ionic_mmap_entry_insert(ctx, rq->q.size, + PHYS_PFN(rq->cmb_addr), + wc ? IONIC_MMAP_WC : 0, + cmb_offset); +} + static void ionic_rq_init_cmb(struct ionic_ibdev *dev, struct ionic_rq *rq, struct ib_udata *udata) @@ -2308,19 +2347,26 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, spin_lock_init(&qp->rq.lock); qp->has_sq = 1; - qp->has_rq = 1; + + if (attr->srq) + qp->srq = to_ionic_srq(attr->srq); + else + qp->srq = NULL; if (attr->qp_type == IB_QPT_GSI) { rc = ionic_get_gsi_qpid(dev, &qp->qpid); } else { udma_mask = BIT(dev->lif_cfg.udma_count) - 1; - if (qp->has_sq) + if (attr->send_cq) udma_mask &= to_ionic_vcq(attr->send_cq)->udma_mask; - if (qp->has_rq) + if (attr->recv_cq) udma_mask &= to_ionic_vcq(attr->recv_cq)->udma_mask; + if (attr->srq) + udma_mask &= BIT(to_ionic_srq(attr->srq)->udma_idx); + if (udata && req.udma_mask) udma_mask &= req.udma_mask; @@ -2347,13 +2393,8 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, goto err_ahid; } - if (udata) { - if (req.rq_cmb & IONIC_CMB_ENABLE) - qp->rq.cmb = req.rq_cmb; - - if (req.sq_cmb & IONIC_CMB_ENABLE) - qp->sq_cmb = req.sq_cmb; - } + if (udata && (req.sq_cmb & IONIC_CMB_ENABLE)) + qp->sq_cmb = req.sq_cmb; rc = ionic_qp_sq_init(dev, ctx, qp, &req.sq, &sq_buf, attr->cap.max_send_wr, attr->cap.max_send_sge, @@ -2361,22 +2402,25 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, if (rc) goto err_sq; - if (qp->has_rq) { - /* for non-srq qps, rq qid is same as qpid */ - qp->rq.qid = qp->qpid; - rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf, - attr->cap.max_recv_wr, attr->cap.max_recv_sge, - req.rq_spec, udata); - if (rc) - goto err_rq; - } else { + if (qp->srq) { rq_buf.tbl_buf = NULL; rq_buf.tbl_limit = 0; rq_buf.tbl_pages = 0; if (udata) rc = ionic_validate_qdesc_zero(&req.rq); + } else { + if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) + qp->rq.cmb = req.rq_cmb; + + /* for non-srq qps, rq qid is same as qpid */ + qp->rq.qid = qp->qpid; + rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf, + attr->cap.max_recv_wr, attr->cap.max_recv_sge, + req.rq_spec, udata); } + if (rc) + goto err_rq; rc = ionic_create_qp_cmd(dev, pd, to_ionic_vcq_cq(attr->send_cq, qp->udma_idx), @@ -2422,35 +2466,9 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, resp.sq_cmb = qp->sq_cmb; } - if (qp->rq.cmb & IONIC_CMB_ENABLE) { - bool wc; + if (!qp->srq && (qp->rq.cmb & IONIC_CMB_ENABLE)) { + ionic_rq_mmap_cmb(dev, ctx, &qp->rq, &resp.rq_cmb_offset); - if ((qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == - (IONIC_CMB_WC | IONIC_CMB_UC)) { - ibdev_dbg(&dev->ibdev, - "Both rq.cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); - qp->rq.cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); - } - - if (qp->rq.cmb & IONIC_CMB_EXPDB) - wc = (qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) - == IONIC_CMB_WC; - else - wc = (qp->rq.cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) - != IONIC_CMB_UC; - - /* let userspace know the mapping */ - if (wc) - qp->rq.cmb |= IONIC_CMB_WC; - else - qp->rq.cmb |= IONIC_CMB_UC; - - qp->rq.mmap_cmb = - ionic_mmap_entry_insert(ctx, - qp->rq.q.size, - PHYS_PFN(qp->rq.cmb_addr), - wc ? IONIC_MMAP_WC : 0, - &resp.rq_cmb_offset); if (!qp->rq.mmap_cmb) { rc = -ENOMEM; goto err_qp_cmd; @@ -2497,7 +2515,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, qp->sq_cqid = cq->cqid; } - if (qp->has_rq) { + if (!qp->srq) { cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx); attr->cap.max_recv_wr = qp->rq.q.mask; @@ -2513,9 +2531,10 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, err_qp_cmd: ionic_destroy_qp_cmd(dev, qp->qpid); err_cmd: - ionic_pgtbl_unbuf(dev, &rq_buf); - if (qp->has_rq) + if (!qp->srq) { + ionic_pgtbl_unbuf(dev, &rq_buf); ionic_rq_destroy(dev, ctx, &qp->rq); + } err_rq: ionic_pgtbl_unbuf(dev, &sq_buf); ionic_qp_sq_destroy(dev, ctx, qp); @@ -2566,7 +2585,7 @@ void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) spin_unlock_irqrestore(&cq->lock, irqflags); } - if (qp->ibqp.recv_cq) { + if (qp->ibqp.recv_cq && !qp->srq) { cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); /* Hold the CQ lock and QP rq.lock to set up flush */ @@ -2642,7 +2661,7 @@ static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) spin_unlock(&qp->sq_lock); } - if (qp->has_rq) { + if (!qp->srq) { spin_lock(&qp->rq.lock); qp->rq.flush = false; qp->rq.q.prod = 0; @@ -2770,7 +2789,7 @@ int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (qp->has_sq) attr->cap.max_send_wr = qp->sq.mask; - if (qp->has_rq) + if (!qp->srq) attr->cap.max_recv_wr = qp->rq.q.mask; init_attr->event_handler = ibqp->event_handler; @@ -2831,7 +2850,7 @@ int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) spin_unlock_irqrestore(&cq->lock, irqflags); } - if (qp->has_rq) + if (!qp->srq) ionic_rq_destroy(dev, ctx, &qp->rq); ionic_qp_sq_destroy(dev, ctx, qp); @@ -2843,3 +2862,267 @@ int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) return 0; } + +static int ionic_create_srq_cmd(struct ionic_ibdev *dev, + struct ionic_ctx *ctx, + struct ionic_srq *srq, + struct ionic_pd *pd, + struct ionic_tbl_buf *buf, + u8 udma_mask) +{ + const u16 flags = (srq->rq.cmb & IONIC_CMB_ENABLE) ? IONIC_SRQF_CMB : 0; + struct ionic_admin_create_srq_resp *resp_buf; + const u16 dbid = ionic_ctx_dbid(dev, ctx); + struct ionic_admin_wr wr = { + .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), + .wqe = { + .op = IONIC_V1_ADMIN_CREATE_SRQ, + .len = cpu_to_le16(IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN), + .cmd.create_srq = { + .pd_id = cpu_to_le32(pd->pdid), + .depth_log2 = srq->rq.q.depth_log2, + .stride_log2 = srq->rq.q.stride_log2, + .page_size_log2 = buf->page_size_log2, + .map_count = cpu_to_le32(buf->tbl_pages), + .dma_addr = ionic_pgtbl_dma(buf, 0), + .dbid = cpu_to_le16(dbid), + .qid = cpu_to_le32(srq->rq.qid), + .low_wqes_limit = cpu_to_le16(srq->srq_limit), + .udma_mask = udma_mask, + .flags = cpu_to_le16(flags), + } + } + }; + dma_addr_t resp_buf_dma; + int rc; + + if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_SRQ) + return -EOPNOTSUPP; + + resp_buf = kzalloc_obj(*resp_buf); + if (!resp_buf) + return -ENOMEM; + + resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, + sizeof(*resp_buf), + DMA_FROM_DEVICE); + + rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); + if (rc) + goto err_dma; + + wr.wqe.cmd.create_srq.resp_dma_addr = cpu_to_le64(resp_buf_dma); + wr.wqe.cmd.create_srq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN); + + ionic_admin_post(dev, &wr); + + rc = ionic_admin_wait(dev, &wr, 0); + + dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), + DMA_FROM_DEVICE); + if (rc) + goto err_dma; + + if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN) { + rc = -EOPNOTSUPP; + goto err_dma; + } + + srq->rq.qid = le32_to_cpu(resp_buf->id); + srq->udma_idx = resp_buf->udma_idx; + +err_dma: + kfree(resp_buf); + + return rc; +} + +static int ionic_destroy_srq_cmd(struct ionic_ibdev *dev, u32 srqid) +{ + struct ionic_admin_wr wr = { + .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), + .wqe = { + .op = IONIC_V1_ADMIN_DESTROY_SRQ, + .len = cpu_to_le16(IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN), + .cmd.destroy_srq = { + .qid = cpu_to_le32(srqid), + }, + } + }; + + if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_SRQ) + return -EOPNOTSUPP; + + ionic_admin_post(dev, &wr); + + return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); +} + +static int ionic_modify_srq_cmd(struct ionic_ibdev *dev, u32 srqid, + u16 srq_limit) +{ + struct ionic_admin_wr wr = { + .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), + .wqe = { + .op = IONIC_V1_ADMIN_MODIFY_SRQ, + .len = cpu_to_le16(IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN), + .cmd.modify_srq = { + .qid = cpu_to_le32(srqid), + .low_wqes_limit = cpu_to_le16(srq_limit), + }, + } + }; + + if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_SRQ) + return -EOPNOTSUPP; + + ionic_admin_post(dev, &wr); + + return ionic_admin_wait(dev, &wr, 0); +} + +int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, + struct ib_udata *udata) +{ + struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); + struct ionic_pd *pd = to_ionic_pd(ibsrq->pd); + struct ionic_srq *srq = to_ionic_srq(ibsrq); + struct ionic_ctx *ctx = + rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); + struct ionic_srq_resp resp = {}; + struct ionic_srq_req req = {}; + struct ionic_tbl_buf buf = {}; + u8 udma_mask; + int rc; + + if (!ctx) + return -EOPNOTSUPP; + + if (attr->srq_type != IB_SRQT_BASIC) + return -EOPNOTSUPP; + + if (attr->attr.max_sge > IONIC_MAX_SRQ_SGES) + return -EINVAL; + + if (attr->attr.srq_limit >= IONIC_MAX_SRQ_LIMIT) + return -EINVAL; + + udma_mask = BIT(dev->lif_cfg.udma_count) - 1; + if (udata) { + rc = ib_copy_validate_udata_in(udata, req, rsvd); + if (rc) + return rc; + + if (memchr_inv(req.rsvd, 0, sizeof(req.rsvd))) + return -EINVAL; + + udma_mask &= req.udma_mask; + } + + if (!udma_mask) + return -EINVAL; + + if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) + srq->rq.cmb = req.rq_cmb; + + rc = ionic_rq_init(dev, ctx, &srq->rq, &req.rq, &buf, + attr->attr.max_wr, attr->attr.max_sge, + req.rq_spec, udata); + if (rc) + return rc; + + srq->srq_limit = attr->attr.srq_limit; + + rc = ionic_create_srq_cmd(dev, ctx, srq, pd, &buf, udma_mask); + if (rc) + goto err_cmd; + + ionic_queue_dbell_init(&srq->rq.q, srq->rq.qid); + + if (udata) { + resp.srqid = srq->rq.qid; + resp.udma_idx = srq->udma_idx; + + if (srq->rq.cmb & IONIC_CMB_ENABLE) { + ionic_rq_mmap_cmb(dev, ctx, &srq->rq, &resp.rq_cmb_offset); + if (!srq->rq.mmap_cmb) { + rc = -ENOMEM; + goto err_srq_cmd; + } + + resp.rq_cmb = srq->rq.cmb; + } + + rc = ib_respond_udata(udata, resp); + if (rc) + goto err_srq_cmd; + } + + ionic_pgtbl_unbuf(dev, &buf); + + attr->attr.max_wr = srq->rq.q.mask; + + return 0; + +err_srq_cmd: + ionic_destroy_srq_cmd(dev, srq->rq.qid); +err_cmd: + ionic_pgtbl_unbuf(dev, &buf); + ionic_rq_destroy(dev, ctx, &srq->rq); + + return rc; +} + +int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +{ + struct ionic_ctx *ctx = + rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); + struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); + struct ionic_srq *srq = to_ionic_srq(ibsrq); + int rc; + + rc = ionic_destroy_srq_cmd(dev, srq->rq.qid); + if (rc) + return rc; + + ionic_rq_destroy(dev, ctx, &srq->rq); + + return 0; +} + +int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + struct ionic_srq *srq = to_ionic_srq(ibsrq); + + srq_attr->max_wr = srq->rq.q.mask; + srq_attr->max_sge = + ionic_v1_recv_wqe_max_sge(srq->rq.q.stride_log2, srq->rq.spec, + srq->rq.cmb & IONIC_CMB_EXPDB); + srq_attr->srq_limit = srq->srq_limit; + + return 0; +} + +int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); + struct ionic_srq *srq = to_ionic_srq(ibsrq); + int rc; + + if (attr_mask & IB_SRQ_MAX_WR) + return -EINVAL; + + if (attr_mask & IB_SRQ_LIMIT) { + if (attr->srq_limit >= IONIC_MAX_SRQ_LIMIT) + return -EINVAL; + + rc = ionic_modify_srq_cmd(dev, srq->rq.qid, attr->srq_limit); + if (rc) + return rc; + + srq->srq_limit = attr->srq_limit; + } + + return 0; +} diff --git a/drivers/infiniband/hw/ionic/ionic_datapath.c b/drivers/infiniband/hw/ionic/ionic_datapath.c index 522c6f08c9fd..c5726c6bd5cb 100644 --- a/drivers/infiniband/hw/ionic/ionic_datapath.c +++ b/drivers/infiniband/hw/ionic/ionic_datapath.c @@ -526,6 +526,12 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev, switch (type) { case IONIC_V1_CQE_TYPE_RECV: + if (qp->srq) { + ibdev_dbg(&dev->ibdev, + "srq recv cqe for qp %u\n", qid); + goto cq_next; + } + spin_lock(&qp->rq.lock); rc = ionic_poll_recv(dev, cq, qp, cqe, wc + npolled); spin_unlock(&qp->rq.lock); @@ -1311,7 +1317,7 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev, if (!bad) return -EINVAL; - if (!qp->has_rq) { + if (qp->srq) { *bad = wr; return -EINVAL; } diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h index b299a1fe5acb..a3991c34aece 100644 --- a/drivers/infiniband/hw/ionic/ionic_fw.h +++ b/drivers/infiniband/hw/ionic/ionic_fw.h @@ -88,6 +88,10 @@ static inline int to_ionic_mr_flags(int access) return flags; } +enum ionic_srq_flags { + IONIC_SRQF_CMB = BIT(0), +}; + enum ionic_qp_flags { /* bits that determine qp access */ IONIC_QPF_REMOTE_WRITE = BIT(0), @@ -857,6 +861,48 @@ struct ionic_admin_query_qp { static_assert(sizeof(struct ionic_admin_query_qp) == IONIC_ADMIN_QUERY_QP_IN_V1_LEN); +struct ionic_admin_create_srq { + __le64 dma_addr; + __le32 map_count; + __le32 pd_id; + __le32 qid; + __le16 dbid; + __le16 low_wqes_limit; + __le16 flags; + __u8 depth_log2; + __u8 stride_log2; + __u8 page_size_log2; + __u8 udma_mask; + __le32 resp_buf_len; + __le64 resp_dma_addr; +} __packed; + +#define IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN 42 +static_assert(sizeof(struct ionic_admin_create_srq) == IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN); + +struct ionic_admin_create_srq_resp { + __le32 id; + __u8 udma_idx; +} __packed; + +#define IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN 5 +static_assert(sizeof(struct ionic_admin_create_srq_resp) == IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN); + +struct ionic_admin_modify_srq { + __le32 qid; + __le16 low_wqes_limit; +} __packed; + +#define IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN 6 +static_assert(sizeof(struct ionic_admin_modify_srq) == IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN); + +struct ionic_admin_destroy_srq { + __le32 qid; +} __packed; + +#define IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN 4 +static_assert(sizeof(struct ionic_admin_destroy_srq) == IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN); + #define ADMIN_WQE_STRIDE 64 #define ADMIN_WQE_HDR_LEN 4 @@ -879,6 +925,9 @@ struct ionic_v1_admin_wqe { struct ionic_admin_destroy_qp destroy_qp; struct ionic_admin_mod_qp mod_qp; struct ionic_admin_query_qp query_qp; + struct ionic_admin_create_srq create_srq; + struct ionic_admin_modify_srq modify_srq; + struct ionic_admin_destroy_srq destroy_srq; } cmd; }; @@ -925,6 +974,10 @@ enum ionic_v1_admin_op { IONIC_V1_ADMIN_DESTROY_AH, IONIC_V1_ADMIN_QP_STATS_HDRS, IONIC_V1_ADMIN_QP_STATS_VALS, + IONIC_V1_ADMIN_CREATE_SRQ = 26, + IONIC_V1_ADMIN_MODIFY_SRQ, + IONIC_V1_ADMIN_QUERY_SRQ, + IONIC_V1_ADMIN_DESTROY_SRQ, IONIC_V1_ADMIN_OPCODES_MAX, }; diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index fa79732c2f7d..8a43ae9451ec 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -73,6 +73,11 @@ static int ionic_query_device(struct ib_device *ibdev, attr->max_ah = dev->lif_cfg.nahs_per_lif; attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2; attr->max_pkeys = IONIC_PKEY_TBL_LEN; + if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) { + attr->max_srq = dev->lif_cfg.srq_count; + attr->max_srq_wr = IONIC_MAX_SRQ_DEPTH; + attr->max_srq_sge = IONIC_MAX_SRQ_SGES; + } return 0; } @@ -266,6 +271,15 @@ static const struct ib_device_ops ionic_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_mw, ionic_mr, ibmw), }; +static const struct ib_device_ops ionic_srq_ops = { + .create_srq = ionic_create_srq, + .modify_srq = ionic_modify_srq, + .query_srq = ionic_query_srq, + .destroy_srq = ionic_destroy_srq, + + INIT_RDMA_OBJ_SIZE(ib_srq, ionic_srq, ibsrq), +}; + static void ionic_init_resids(struct ionic_ibdev *dev) { ionic_resid_init(&dev->inuse_cqid, dev->lif_cfg.cq_count); @@ -354,6 +368,8 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) goto err_admin; ib_set_device_ops(&dev->ibdev, &ionic_dev_ops); + if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) + ib_set_device_ops(&dev->ibdev, &ionic_srq_ops); ionic_stats_init(dev); diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h index aa3b4122c5ec..3ee7416427fd 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.h +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -36,6 +36,10 @@ #define IONIC_SQCMB_ORDER 5 #define IONIC_RQCMB_ORDER 0 +#define IONIC_MAX_SRQ_SGES 2 +#define IONIC_MAX_SRQ_LIMIT 0xffff +#define IONIC_MAX_SRQ_DEPTH 0xffff + #define IONIC_META_LAST ((void *)1ul) #define IONIC_META_POSTED ((void *)2ul) @@ -263,6 +267,13 @@ struct ionic_rq { bool flush; }; +struct ionic_srq { + struct ib_srq ibsrq; + struct ionic_rq rq; + u16 srq_limit; + u8 udma_idx; +}; + struct ionic_qp { struct ib_qp ibqp; enum ib_qp_state state; @@ -273,7 +284,6 @@ struct ionic_qp { u8 udma_idx; u8 has_ah:1; u8 has_sq:1; - u8 has_rq:1; u8 sig_all:1; struct list_head qp_list_counter; @@ -311,6 +321,7 @@ struct ionic_qp { int dcqcn_profile; struct ib_ud_header *hdr; + struct ionic_srq *srq; }; struct ionic_ah { @@ -405,6 +416,11 @@ static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp) return container_of(ibqp, struct ionic_qp, ibqp); } +static inline struct ionic_srq *to_ionic_srq(struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct ionic_srq, ibsrq); +} + static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah) { return container_of(ibah, struct ionic_ah, ibah); @@ -513,6 +529,12 @@ int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_qp_init_attr *init_attr); int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); +int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, + struct ib_udata *udata); +int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); +int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); +int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); /* ionic_datapath.c */ int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c index c381bda4360d..66cee57442e7 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c @@ -75,6 +75,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) cfg->eq_count = lif->ionic->neqs_per_lif; cfg->cq_count = le32_to_cpu(ident->rdma.cq_qtype.qid_count); cfg->qp_count = le32_to_cpu(ident->rdma.sq_qtype.qid_count); + cfg->srq_count = le32_to_cpu(ident->rdma.srq_qtype.qid_count); cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif); cfg->aq_qtype = ident->rdma.aq_qtype.qtype; diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h index 68aec06908ca..bc14fc0921da 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h @@ -38,6 +38,7 @@ struct ionic_lif_cfg { int eq_count; int cq_count; int qp_count; + int srq_count; u16 stats_type; u8 aq_qtype; diff --git a/include/uapi/rdma/ionic-abi.h b/include/uapi/rdma/ionic-abi.h index 3d8d56922a93..ac48fcd19a4f 100644 --- a/include/uapi/rdma/ionic-abi.h +++ b/include/uapi/rdma/ionic-abi.h @@ -113,7 +113,7 @@ struct ionic_srq_req { }; struct ionic_srq_resp { - __u32 qpid; + __u32 srqid; __u8 rq_cmb; __u8 udma_idx; __u8 rsvd[2]; From 4dd2160af277acf8f2897a2ec7df4352a7ca50bd Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 30 Sep 2026 14:51:11 +0530 Subject: [PATCH 110/153] RDMA/ionic: implement SRQ event handling support Introduce SRQ event types and associated events. Report these events through the SRQ event handler during event processing. Signed-off-by: Amit Kumar Signed-off-by: Abhijit Gangurde Link: https://patch.msgid.link/20260930092111.3176738-6-abhijit.gangurde@amd.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/ionic/ionic_admin.c | 47 +++++++++++++++++++ .../infiniband/hw/ionic/ionic_controlpath.c | 12 +++++ drivers/infiniband/hw/ionic/ionic_fw.h | 6 ++- drivers/infiniband/hw/ionic/ionic_ibdev.c | 4 ++ drivers/infiniband/hw/ionic/ionic_ibdev.h | 10 ++++ 5 files changed, 78 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/ionic/ionic_admin.c b/drivers/infiniband/hw/ionic/ionic_admin.c index 93c01bd40d78..0ccd869ad8e6 100644 --- a/drivers/infiniband/hw/ionic/ionic_admin.c +++ b/drivers/infiniband/hw/ionic/ionic_admin.c @@ -945,6 +945,49 @@ static void ionic_qp_event(struct ionic_ibdev *dev, u32 qpid, u8 code) kref_put(&qp->qp_kref, ionic_qp_complete); } +static void ionic_srq_event(struct ionic_ibdev *dev, u32 srqid, u8 code) +{ + unsigned long irqflags; + struct ionic_srq *srq; + struct ib_event ibev; + + xa_lock_irqsave(&dev->srq_tbl, irqflags); + srq = xa_load(&dev->srq_tbl, srqid); + if (srq) + kref_get(&srq->kref); + xa_unlock_irqrestore(&dev->srq_tbl, irqflags); + + if (!srq) { + ibdev_dbg(&dev->ibdev, + "missing srqid %#x code %u\n", srqid, code); + return; + } + + ibev.device = &dev->ibdev; + ibev.element.srq = &srq->ibsrq; + + switch (code) { + case IONIC_V1_EQE_SRQ_LIMIT_REACHED: + ibev.event = IB_EVENT_SRQ_LIMIT_REACHED; + break; + + case IONIC_V1_EQE_SRQ_ERR: + ibev.event = IB_EVENT_SRQ_ERR; + break; + + default: + ibdev_dbg(&dev->ibdev, + "unrecognized srqid %#x code %u\n", srqid, code); + goto out; + } + + if (srq->ibsrq.event_handler) + srq->ibsrq.event_handler(&ibev, srq->ibsrq.srq_context); + +out: + kref_put(&srq->kref, ionic_srq_complete); +} + static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget) { struct ionic_ibdev *dev = eq->dev; @@ -978,6 +1021,10 @@ static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget) ionic_qp_event(dev, qid, code); break; + case IONIC_V1_EQE_TYPE_SRQ: + ionic_srq_event(dev, qid, code); + break; + default: ibdev_dbg(&dev->ibdev, "unknown event %#x type %u\n", evt, type); diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index 3e7354361e40..18ad967812f5 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -3058,6 +3058,13 @@ int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, goto err_srq_cmd; } + kref_init(&srq->kref); + init_completion(&srq->rel_comp); + + rc = xa_insert_irq(&dev->srq_tbl, srq->rq.qid, srq, GFP_KERNEL); + if (rc) + goto err_srq_cmd; + ionic_pgtbl_unbuf(dev, &buf); attr->attr.max_wr = srq->rq.q.mask; @@ -3081,6 +3088,11 @@ int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) struct ionic_srq *srq = to_ionic_srq(ibsrq); int rc; + if (xa_erase_irq(&dev->srq_tbl, srq->rq.qid)) { + kref_put(&srq->kref, ionic_srq_complete); + wait_for_completion(&srq->rel_comp); + } + rc = ionic_destroy_srq_cmd(dev, srq->rq.qid); if (rc) return rc; diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h index a3991c34aece..1db67fcb9a53 100644 --- a/drivers/infiniband/hw/ionic/ionic_fw.h +++ b/drivers/infiniband/hw/ionic/ionic_fw.h @@ -1013,7 +1013,7 @@ enum ionic_v1_eqe_evt_bits { /* cq error events */ IONIC_V1_EQE_CQ_ERR = 8, - /* qp and srq events */ + /* qp events */ IONIC_V1_EQE_TYPE_QP = 1, /* qp normal events */ IONIC_V1_EQE_SRQ_LEVEL = 0, @@ -1024,6 +1024,10 @@ enum ionic_v1_eqe_evt_bits { IONIC_V1_EQE_QP_ERR = 8, IONIC_V1_EQE_QP_ERR_REQUEST = 9, IONIC_V1_EQE_QP_ERR_ACCESS = 10, + /* srq events */ + IONIC_V1_EQE_TYPE_SRQ = 2, + IONIC_V1_EQE_SRQ_LIMIT_REACHED = 0, + IONIC_V1_EQE_SRQ_ERR = 1, }; enum ionic_tfp_csum_profiles { diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index 8a43ae9451ec..b4d4544bb7ac 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -318,6 +318,8 @@ static void ionic_destroy_ibdev(struct ionic_ibdev *dev) xa_destroy(&dev->qp_tbl); WARN_ON(!xa_empty(&dev->cq_tbl)); xa_destroy(&dev->cq_tbl); + WARN_ON(!xa_empty(&dev->srq_tbl)); + xa_destroy(&dev->srq_tbl); ib_dealloc_device(&dev->ibdev); } @@ -336,6 +338,7 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) xa_init_flags(&dev->qp_tbl, XA_FLAGS_LOCK_IRQ); xa_init_flags(&dev->cq_tbl, XA_FLAGS_LOCK_IRQ); + xa_init_flags(&dev->srq_tbl, XA_FLAGS_LOCK_IRQ); ionic_init_resids(dev); @@ -388,6 +391,7 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) ionic_destroy_resids(dev); xa_destroy(&dev->qp_tbl); xa_destroy(&dev->cq_tbl); + xa_destroy(&dev->srq_tbl); ib_dealloc_device(&dev->ibdev); return ERR_PTR(rc); diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h index 3ee7416427fd..f8d580dd4d95 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.h +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -92,6 +92,7 @@ struct ionic_ibdev { struct xarray qp_tbl; struct xarray cq_tbl; + struct xarray srq_tbl; struct ionic_resid_bits inuse_dbid; struct ionic_resid_bits inuse_pdid; @@ -270,6 +271,8 @@ struct ionic_rq { struct ionic_srq { struct ib_srq ibsrq; struct ionic_rq rq; + struct kref kref; + struct completion rel_comp; u16 srq_limit; u8 udma_idx; }; @@ -466,6 +469,13 @@ static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev, return dev->lif_cfg.alloc_qid_cap & qtype; } +static inline void ionic_srq_complete(struct kref *kref) +{ + struct ionic_srq *srq = container_of(kref, struct ionic_srq, kref); + + complete(&srq->rel_comp); +} + /* ionic_admin.c */ extern struct workqueue_struct *ionic_evt_workq; void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); From a727a424068b5d54fe80ad1cfbf2fa6be2f39e61 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:18 +0300 Subject: [PATCH 111/153] RDMA/mlx5: Notify data direct users via a notifier chain on unbind Currently during unbind, data direct code calls mlx5_ib_data_direct_unbind() directly. To make the unbind notification generic and support multiple users, deliver an MLX5_DATA_DIRECT_UNBIND event through a blocking_notifier chain kept per registration. Users register a notifier_block via mlx5_data_direct_ib_reg(); mlx5_ib embeds one in struct mlx5_ib_dev and, from its notifier callback, runs the existing mlx5_ib_data_direct_unbind() teardown. This patch doesn't introduce any functional changes. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-2-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 17 ++++++++++++++--- drivers/infiniband/hw/mlx5/data_direct.h | 12 ++++++++++-- drivers/infiniband/hw/mlx5/main.c | 17 +++++++++++++++-- drivers/infiniband/hw/mlx5/mlx5_ib.h | 1 + 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index d57484245c38..04affdbdcb12 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -4,6 +4,9 @@ */ #include "mlx5_ib.h" + +#include + #include "data_direct.h" static LIST_HEAD(mlx5_data_direct_dev_list); @@ -18,6 +21,7 @@ struct mlx5_data_direct_registration { struct mlx5_ib_dev *ibdev; char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; struct list_head list; + struct blocking_notifier_head users; }; static const struct pci_device_id mlx5_data_direct_pci_table[] = { @@ -78,7 +82,8 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) return 0; } -int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid) +int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, + struct notifier_block *nb) { struct mlx5_data_direct_registration *reg; struct mlx5_data_direct_dev *dev; @@ -89,6 +94,8 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid) reg->ibdev = ibdev; strscpy(reg->vuid, vuid); + BLOCKING_INIT_NOTIFIER_HEAD(®->users); + blocking_notifier_chain_register(®->users, nb); mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { @@ -106,13 +113,15 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid) return 0; } -void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev) +void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb) { struct mlx5_data_direct_registration *reg; mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (reg->ibdev == ibdev) { + blocking_notifier_chain_unregister(®->users, nb); list_del(®->list); kfree(reg); goto end; @@ -150,7 +159,9 @@ static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev) list_del(&dev->list); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_ib_data_direct_unbind(reg->ibdev); + blocking_notifier_call_chain(®->users, + MLX5_DATA_DIRECT_UNBIND, + NULL); } mutex_unlock(&mlx5_data_direct_mutex); } diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index 2fd2bdbe8f69..a9bb471949e2 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -6,8 +6,14 @@ #ifndef _MLX5_IB_DATA_DIRECT_H #define _MLX5_IB_DATA_DIRECT_H +#include + struct mlx5_ib_dev; +enum mlx5_data_direct_event { + MLX5_DATA_DIRECT_UNBIND, +}; + struct mlx5_data_direct_dev { struct device *device; struct pci_dev *pdev; @@ -15,8 +21,10 @@ struct mlx5_data_direct_dev { struct list_head list; }; -int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid); -void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev); +int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, + struct notifier_block *nb); +void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb); int mlx5_data_direct_driver_register(void); void mlx5_data_direct_driver_unregister(void); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 02809114fc79..547275dfa40e 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -4019,6 +4019,18 @@ static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev, return false; } +static int mlx5_ib_data_direct_event(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct mlx5_ib_dev *dev = + container_of(nb, struct mlx5_ib_dev, data_direct_nb); + + if (action == MLX5_DATA_DIRECT_UNBIND) + mlx5_ib_data_direct_unbind(dev); + + return NOTIFY_OK; +} + static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) { char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {}; @@ -4037,7 +4049,8 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) return ret; INIT_LIST_HEAD(&dev->data_direct_mr_list); - ret = mlx5_data_direct_ib_reg(dev, vuid); + dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event; + ret = mlx5_data_direct_ib_reg(dev, vuid, &dev->data_direct_nb); if (ret) mlx5_ib_free_data_direct_resources(dev); @@ -4050,7 +4063,7 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) !MLX5_CAP_GEN_2(dev->mdev, query_vuid)) return; - mlx5_data_direct_ib_unreg(dev); + mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb); mlx5_ib_free_data_direct_resources(dev); } diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 522984d958bb..97d2aa79e95d 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1131,6 +1131,7 @@ struct mlx5_ib_dev { spinlock_t reset_flow_resource_lock; struct list_head qp_list; struct list_head data_direct_mr_list; + struct notifier_block data_direct_nb; /* Array with num_ports elements */ struct mlx5_ib_port *port; struct mlx5_sq_bfreg bfreg; From 2365f8d7f609cff766b150028372bc28d8b436f7 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:19 +0300 Subject: [PATCH 112/153] RDMA/mlx5: Add mlx5_data_direct_supported() helper The data direct capability FW checks are duplicated inline in mlx5_ib_data_direct_init() and mlx5_ib_data_direct_cleanup(). Wrap it in a mlx5_data_direct_supported() helper and use it in both places, in preparation for moving the data direct matching code to mlx5_core. For the same reason put it in driver.h instead of data_direct.h. No functional change. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-3-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/main.c | 6 ++---- include/linux/mlx5/driver.h | 6 ++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 547275dfa40e..0c57cb39ae7e 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -4036,8 +4036,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {}; int ret; - if (!MLX5_CAP_GEN(dev->mdev, data_direct) || - !MLX5_CAP_GEN_2(dev->mdev, query_vuid)) + if (!mlx5_data_direct_supported(dev->mdev)) return 0; ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid); @@ -4059,8 +4058,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) { - if (!MLX5_CAP_GEN(dev->mdev, data_direct) || - !MLX5_CAP_GEN_2(dev->mdev, query_vuid)) + if (!mlx5_data_direct_supported(dev->mdev)) return; mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index b1871c0821d0..ee29f95ead18 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1395,4 +1395,10 @@ static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev) #define MLX5_SW_IMAGE_GUID_MAX_BYTES 9 +static inline bool mlx5_data_direct_supported(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, data_direct) && + MLX5_CAP_GEN_2(mdev, query_vuid); +} + #endif /* MLX5_DRIVER_H */ From ebbb9f87689998157e7310e96f585bc293f6fcb0 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:20 +0300 Subject: [PATCH 113/153] RDMA/mlx5: Make the data direct VUID query self-contained mlx5_cmd_query_vuid() in cmd.c is generic over the data_direct flag but its only caller is the data direct init path, which always passes data_direct=true. Move it into data_direct.c as mlx5_data_direct_query_vuid(), dropping the always-true parameter and hardcoding data_direct=1. This groups the query with the rest of the data direct code so that it travels together when the data direct matching code is relocated to mlx5_core, and lets it become static there once the query is folded into the core bind path. This function will become private in a subsequent patch. No functional change. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-4-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/cmd.c | 21 --------------------- drivers/infiniband/hw/mlx5/cmd.h | 2 -- drivers/infiniband/hw/mlx5/data_direct.c | 20 ++++++++++++++++++++ drivers/infiniband/hw/mlx5/data_direct.h | 3 +++ drivers/infiniband/hw/mlx5/main.c | 3 ++- include/linux/mlx5/data_direct.h | 13 +++++++++++++ 6 files changed, 38 insertions(+), 24 deletions(-) create mode 100644 include/linux/mlx5/data_direct.h diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c index 7c08e3008927..895b62cc528d 100644 --- a/drivers/infiniband/hw/mlx5/cmd.c +++ b/drivers/infiniband/hw/mlx5/cmd.c @@ -245,24 +245,3 @@ int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid) MLX5_SET(dealloc_uar_in, in, uid, uid); return mlx5_cmd_exec_in(dev, dealloc_uar, in); } - -int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct, - char *out_vuid) -{ - u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + - MLX5_ST_SZ_BYTES(array1024_auto)] = {}; - u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {}; - char *vuid; - int err; - - MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID); - MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id)); - MLX5_SET(query_vuid_in, in, data_direct, data_direct); - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); - if (err) - return err; - - vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid); - memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto)); - return 0; -} diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h index e6c88b6ebd0d..e5cd31270443 100644 --- a/drivers/infiniband/hw/mlx5/cmd.h +++ b/drivers/infiniband/hw/mlx5/cmd.h @@ -58,6 +58,4 @@ int mlx5_cmd_mad_ifc(struct mlx5_ib_dev *dev, const void *inb, void *outb, u16 opmod, u8 port); int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid); int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid); -int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct, - char *out_vuid); #endif /* MLX5_IB_CMD_H */ diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index 04affdbdcb12..8d006b611c63 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -24,6 +24,26 @@ struct mlx5_data_direct_registration { struct blocking_notifier_head users; }; +int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid) +{ + u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + + MLX5_ST_SZ_BYTES(array1024_auto)] = {}; + u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {}; + char *vuid; + int err; + + MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID); + MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id)); + MLX5_SET(query_vuid_in, in, data_direct, 1); + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid); + memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto)); + return 0; +} + static const struct pci_device_id mlx5_data_direct_pci_table[] = { { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */ { 0, } diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index a9bb471949e2..fa3a145c06c6 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -7,7 +7,9 @@ #define _MLX5_IB_DATA_DIRECT_H #include +#include +struct mlx5_core_dev; struct mlx5_ib_dev; enum mlx5_data_direct_event { @@ -21,6 +23,7 @@ struct mlx5_data_direct_dev { struct list_head list; }; +int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, struct notifier_block *nb); void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev, diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 0c57cb39ae7e..8d6914de1442 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -4039,7 +4040,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) if (!mlx5_data_direct_supported(dev->mdev)) return 0; - ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid); + ret = mlx5_data_direct_query_vuid(dev->mdev, vuid); if (ret) return ret; diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h new file mode 100644 index 000000000000..4556e58c6d03 --- /dev/null +++ b/include/linux/mlx5/data_direct.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#ifndef _MLX5_DATA_DIRECT_H +#define _MLX5_DATA_DIRECT_H + +struct mlx5_core_dev; + +int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); + +#endif From 0a4095faca722b7f5451edd7e9766dff4e15accd Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:21 +0300 Subject: [PATCH 114/153] RDMA/mlx5: Give the data direct PCI driver a proper name The data direct PCI driver uses KBUILD_MODNAME as its driver name, which resolves to "mlx5_ib". This is misleading (the driver binds the data direct PCIe function, not an IB device) and would silently change to "mlx5_core" when the driver is relocated there. Name the driver explicitly "mlx5_dd". Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-5-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index 8d006b611c63..a80778f2b9b2 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -240,7 +240,7 @@ static void mlx5_data_direct_remove(struct pci_dev *pdev) } static struct pci_driver mlx5_data_direct_driver = { - .name = KBUILD_MODNAME, + .name = "mlx5_dd", .id_table = mlx5_data_direct_pci_table, .probe = mlx5_data_direct_probe, .remove = mlx5_data_direct_remove, From 47723d4febd62be03e1f392ddf3b54506f9d8397 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:22 +0300 Subject: [PATCH 115/153] RDMA/mlx5: Move and rename data direct resource functions Move the data direct resource creation/deletion and resource struct to be self contained in the data direct file. Rename the functions to be consistent with the rest. While at it clear the fields on resource deletion. This is preparation for moving the data direct code to the mlx5_core driver. The code has no functional changes. Keep the resource struct and function declarations in the IB-local data_direct.h header. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-6-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 80 ++++++++++++++++++++++ drivers/infiniband/hw/mlx5/data_direct.h | 10 +++ drivers/infiniband/hw/mlx5/main.c | 87 +----------------------- drivers/infiniband/hw/mlx5/mlx5_ib.h | 8 +-- 4 files changed, 94 insertions(+), 91 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index a80778f2b9b2..cc3f741b4f81 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -102,6 +102,86 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) return 0; } +int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) +{ + int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); + struct mlx5_core_dev *mdev = dev->mdev; + bool ro_supp = false; + void *mkc; + u32 mkey; + u32 pdn; + u32 *in; + int err; + + err = mlx5_core_alloc_pd(mdev, &pdn); + if (err) + return err; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err; + } + + MLX5_SET(create_mkey_in, in, data_direct, 1); + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); + MLX5_SET(mkc, mkc, lw, 1); + MLX5_SET(mkc, mkc, lr, 1); + MLX5_SET(mkc, mkc, rw, 1); + MLX5_SET(mkc, mkc, rr, 1); + MLX5_SET(mkc, mkc, a, 1); + MLX5_SET(mkc, mkc, pd, pdn); + MLX5_SET(mkc, mkc, length64, 1); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); + if (err) + goto err_mkey; + + dev->ddr.mkey = mkey; + dev->ddr.pdn = pdn; + + /* create another mkey with RO support */ + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) { + MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); + ro_supp = true; + } + + if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) { + MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); + ro_supp = true; + } + + if (ro_supp) { + err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); + /* RO is defined as best effort */ + if (!err) { + dev->ddr.mkey_ro = mkey; + dev->ddr.mkey_ro_valid = true; + } + } + + kvfree(in); + return 0; + +err_mkey: + kvfree(in); +err: + mlx5_core_dealloc_pd(mdev, pdn); + return err; +} + +void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) +{ + if (dev->ddr.mkey_ro_valid) + mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro); + + mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey); + mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn); + + memset(&dev->ddr, 0, sizeof(dev->ddr)); +} + int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, struct notifier_block *nb) { diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index fa3a145c06c6..90e0565d7333 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -23,6 +23,16 @@ struct mlx5_data_direct_dev { struct list_head list; }; +struct mlx5_data_direct_resources { + u32 pdn; + u32 mkey; + u32 mkey_ro; + u8 mkey_ro_valid :1; +}; + +int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev); +void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev); + int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, struct notifier_block *nb); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 8d6914de1442..0bece3e55109 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3474,87 +3474,6 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev) mutex_destroy(&devr->srq_lock); } -static int -mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev) -{ - int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); - struct mlx5_core_dev *mdev = dev->mdev; - bool ro_supp = false; - void *mkc; - u32 mkey; - u32 pdn; - u32 *in; - int err; - - err = mlx5_core_alloc_pd(mdev, &pdn); - if (err) - return err; - - in = kvzalloc(inlen, GFP_KERNEL); - if (!in) { - err = -ENOMEM; - goto err; - } - - MLX5_SET(create_mkey_in, in, data_direct, 1); - mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); - MLX5_SET(mkc, mkc, lw, 1); - MLX5_SET(mkc, mkc, lr, 1); - MLX5_SET(mkc, mkc, rw, 1); - MLX5_SET(mkc, mkc, rr, 1); - MLX5_SET(mkc, mkc, a, 1); - MLX5_SET(mkc, mkc, pd, pdn); - MLX5_SET(mkc, mkc, length64, 1); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); - if (err) - goto err_mkey; - - dev->ddr.mkey = mkey; - dev->ddr.pdn = pdn; - - /* create another mkey with RO support */ - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) { - MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); - ro_supp = true; - } - - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) { - MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); - ro_supp = true; - } - - if (ro_supp) { - err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); - /* RO is defined as best effort */ - if (!err) { - dev->ddr.mkey_ro = mkey; - dev->ddr.mkey_ro_valid = true; - } - } - - kvfree(in); - return 0; - -err_mkey: - kvfree(in); -err: - mlx5_core_dealloc_pd(mdev, pdn); - return err; -} - -static void -mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev) -{ - - if (dev->ddr.mkey_ro_valid) - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro); - - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey); - mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn); -} - static u32 get_core_cap_flags(struct ib_device *ibdev, struct mlx5_hca_vport_context *rep) { @@ -4044,7 +3963,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) if (ret) return ret; - ret = mlx5_ib_create_data_direct_resources(dev); + ret = mlx5_data_direct_create_resources(dev); if (ret) return ret; @@ -4052,7 +3971,7 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event; ret = mlx5_data_direct_ib_reg(dev, vuid, &dev->data_direct_nb); if (ret) - mlx5_ib_free_data_direct_resources(dev); + mlx5_data_direct_free_resources(dev); return ret; } @@ -4063,7 +3982,7 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) return; mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb); - mlx5_ib_free_data_direct_resources(dev); + mlx5_data_direct_free_resources(dev); } static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 97d2aa79e95d..ff026985e293 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -24,6 +24,7 @@ #include #include +#include "data_direct.h" #include "srq.h" #include "qp.h" #include "macsec.h" @@ -781,13 +782,6 @@ struct mlx5_ib_port_resources { struct work_struct pkey_change_work; }; -struct mlx5_data_direct_resources { - u32 pdn; - u32 mkey; - u32 mkey_ro; - u8 mkey_ro_valid :1; -}; - struct mlx5_ib_resources { struct ib_cq *c0; struct mutex cq_lock; From 1c879b1e9f2ed620523e17ca239683bc8029e143 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:23 +0300 Subject: [PATCH 116/153] RDMA/mlx5: Add new registration stage for data direct users For the upcoming move of data direct to mlx5_core driver, the API needs to be split into init and register stages. That's because only the core will do the init phase but both IB and netdev devices will need to register. While data direct still lives in mlx5_ib, do all steps in mlx5_ib_data_direct_init(). While at it, make mlx5_data_direct_query_vuid() private as it is no longer needed. This patch has no functional changes. It is a preparation for moving data direct infrastructure to mlx5_core. For this reason the new functions are added to the new header. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-7-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 118 ++++++++++++++++++----- drivers/infiniband/hw/mlx5/data_direct.h | 5 - drivers/infiniband/hw/mlx5/main.c | 41 ++++---- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 - include/linux/mlx5/data_direct.h | 10 +- 5 files changed, 120 insertions(+), 57 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index cc3f741b4f81..f871e3ce9f73 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -24,7 +24,8 @@ struct mlx5_data_direct_registration { struct blocking_notifier_head users; }; -int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid) +static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, + char *out_vuid) { u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + MLX5_ST_SZ_BYTES(array1024_auto)] = {}; @@ -182,25 +183,65 @@ void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) memset(&dev->ddr, 0, sizeof(dev->ddr)); } -int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, - struct notifier_block *nb) +static struct mlx5_data_direct_registration * +mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev) +{ + struct mlx5_data_direct_registration *reg; + + list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) + if (reg->ibdev == ibdev) + return reg; + return NULL; +} + +static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev, + struct mlx5_data_direct_dev *dev) +{ + mutex_lock(&ibdev->data_direct_lock); + ibdev->data_direct_dev = dev; + mutex_unlock(&ibdev->data_direct_lock); +} + +static void +mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) +{ + struct mlx5_ib_dev *ibdev = reg->ibdev; + + mutex_lock(&ibdev->data_direct_lock); + blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, + NULL); + ibdev->data_direct_dev = NULL; + mutex_unlock(&ibdev->data_direct_lock); +} + +int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) { struct mlx5_data_direct_registration *reg; struct mlx5_data_direct_dev *dev; + int err; + + if (!mlx5_data_direct_supported(ibdev->mdev)) + return 0; reg = kzalloc_obj(*reg); if (!reg) return -ENOMEM; reg->ibdev = ibdev; - strscpy(reg->vuid, vuid); BLOCKING_INIT_NOTIFIER_HEAD(®->users); - blocking_notifier_chain_register(®->users, nb); + + err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid); + if (err) { + mlx5_ib_warn(ibdev, "Failed to query VUID, disabling data direct, err=%d\n", + err); + kfree(reg); + return err; + } mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { - if (strcmp(dev->vuid, vuid) == 0) { - mlx5_ib_data_direct_bind(ibdev, dev); + if (strcmp(dev->vuid, reg->vuid) == 0) { + mlx5_data_direct_bind(ibdev, dev); break; } } @@ -213,25 +254,56 @@ int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, return 0; } -void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev, - struct notifier_block *nb) +void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) { struct mlx5_data_direct_registration *reg; - mutex_lock(&mlx5_data_direct_mutex); - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { - if (reg->ibdev == ibdev) { - blocking_notifier_chain_unregister(®->users, nb); - list_del(®->list); - kfree(reg); - goto end; - } - } + if (!mlx5_data_direct_supported(ibdev->mdev)) + return; - WARN_ON(true); -end: + mutex_lock(&mlx5_data_direct_mutex); + reg = mlx5_data_direct_get_reg(ibdev); + if (reg) { + list_del(®->list); + mlx5_data_direct_do_unbind(reg); + } + mutex_unlock(&mlx5_data_direct_mutex); + + kfree(reg); +} + +int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(ibdev->mdev)) + return 0; + + mutex_lock(&mlx5_data_direct_mutex); + reg = mlx5_data_direct_get_reg(ibdev); + if (reg) + blocking_notifier_chain_register(®->users, nb); + mutex_unlock(&mlx5_data_direct_mutex); + return 0; +} +EXPORT_SYMBOL_GPL(mlx5_data_direct_register); + +void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(ibdev->mdev)) + return; + + mutex_lock(&mlx5_data_direct_mutex); + reg = mlx5_data_direct_get_reg(ibdev); + if (reg) + blocking_notifier_chain_unregister(®->users, nb); mutex_unlock(&mlx5_data_direct_mutex); } +EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) { @@ -240,7 +312,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_ib_data_direct_bind(reg->ibdev, dev); + mlx5_data_direct_bind(reg->ibdev, dev); } /* Add the data direct device to the global list, further IB devices may @@ -259,9 +331,7 @@ static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev) list_del(&dev->list); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) - blocking_notifier_call_chain(®->users, - MLX5_DATA_DIRECT_UNBIND, - NULL); + mlx5_data_direct_do_unbind(reg); } mutex_unlock(&mlx5_data_direct_mutex); } diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index 90e0565d7333..f207239461bf 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -33,11 +33,6 @@ struct mlx5_data_direct_resources { int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev); void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev); -int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); -int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid, - struct notifier_block *nb); -void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev, - struct notifier_block *nb); int mlx5_data_direct_driver_register(void); void mlx5_data_direct_driver_unregister(void); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 0bece3e55109..d0141509870a 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3945,33 +3945,41 @@ static int mlx5_ib_data_direct_event(struct notifier_block *nb, struct mlx5_ib_dev *dev = container_of(nb, struct mlx5_ib_dev, data_direct_nb); - if (action == MLX5_DATA_DIRECT_UNBIND) - mlx5_ib_data_direct_unbind(dev); + if (action != MLX5_DATA_DIRECT_UNBIND) + return NOTIFY_DONE; + + mlx5_ib_revoke_data_direct_mrs(dev); return NOTIFY_OK; } static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) { - char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {}; int ret; if (!mlx5_data_direct_supported(dev->mdev)) return 0; - ret = mlx5_data_direct_query_vuid(dev->mdev, vuid); + ret = mlx5_data_direct_init(dev); if (ret) return ret; ret = mlx5_data_direct_create_resources(dev); if (ret) - return ret; + goto err_resources; INIT_LIST_HEAD(&dev->data_direct_mr_list); dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event; - ret = mlx5_data_direct_ib_reg(dev, vuid, &dev->data_direct_nb); + ret = mlx5_data_direct_register(dev, &dev->data_direct_nb); if (ret) - mlx5_data_direct_free_resources(dev); + goto err_register; + + return ret; + +err_register: + mlx5_data_direct_free_resources(dev); +err_resources: + mlx5_data_direct_cleanup(dev); return ret; } @@ -3981,8 +3989,9 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) if (!mlx5_data_direct_supported(dev->mdev)) return; - mlx5_data_direct_ib_unreg(dev, &dev->data_direct_nb); + mlx5_data_direct_unregister(dev, &dev->data_direct_nb); mlx5_data_direct_free_resources(dev); + mlx5_data_direct_cleanup(dev); } static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) @@ -5008,22 +5017,6 @@ static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev) cancel_work_sync(&devr->ports[port].pkey_change_work); } -void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev, - struct mlx5_data_direct_dev *dev) -{ - mutex_lock(&ibdev->data_direct_lock); - ibdev->data_direct_dev = dev; - mutex_unlock(&ibdev->data_direct_lock); -} - -void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev) -{ - mutex_lock(&ibdev->data_direct_lock); - mlx5_ib_revoke_data_direct_mrs(ibdev); - ibdev->data_direct_dev = NULL; - mutex_unlock(&ibdev->data_direct_lock); -} - void __mlx5_ib_remove(struct mlx5_ib_dev *dev, const struct mlx5_ib_profile *profile, int stage) diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index ff026985e293..e56f3eee7ba1 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1396,9 +1396,6 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table); struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, struct ib_dm_mr_attr *attr, struct uverbs_attr_bundle *attrs); -void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev, - struct mlx5_data_direct_dev *dev); -void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev); void mlx5_ib_revoke_data_direct_mrs(struct mlx5_ib_dev *dev); #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h index 4556e58c6d03..71f47547aebb 100644 --- a/include/linux/mlx5/data_direct.h +++ b/include/linux/mlx5/data_direct.h @@ -7,7 +7,15 @@ #define _MLX5_DATA_DIRECT_H struct mlx5_core_dev; +struct mlx5_ib_dev; +struct notifier_block; -int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, char *out_vuid); +int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev); +void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev); + +int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb); +void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, + struct notifier_block *nb); #endif From 26f27809b0034e5df3ae201659604b39c4ff88ab Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:24 +0300 Subject: [PATCH 117/153] RDMA/mlx5: Extract IB specific lock out of data direct The data_direct_lock from struct mlx5_ib_dev is an IB specific lock. It protects from unbind while mlx5_data_direct_dev is being accessed. So move data_direct_lock out of the generic bind/unbind functions and into the IB specific notifier callback. data_direct_dev can be set during bind/unbind with a simple WRITE_ONCE, the lock is no longer needed. Each user of the data direct registration API has to do their own locking if necessary. During unbind, the data_direct_dev is set to NULL before calling the notifiers to avoid multiple users still accessing the data_direct_dev while unbind is in progress. This is to avoid another data direct specific lock which can lead to lock ordering issues. This patch has no functional changes. Signed-off-by: Dragos Tatulea Reviewed-by: Leon Romanovsky Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-8-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 8 ++------ drivers/infiniband/hw/mlx5/main.c | 2 ++ drivers/infiniband/hw/mlx5/mlx5_ib.h | 4 +++- drivers/infiniband/hw/mlx5/mr.c | 2 +- drivers/infiniband/hw/mlx5/std_types.c | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index f871e3ce9f73..16afacdd3601 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -197,9 +197,7 @@ mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev) static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev, struct mlx5_data_direct_dev *dev) { - mutex_lock(&ibdev->data_direct_lock); - ibdev->data_direct_dev = dev; - mutex_unlock(&ibdev->data_direct_lock); + WRITE_ONCE(ibdev->data_direct_dev, dev); } static void @@ -207,11 +205,9 @@ mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) { struct mlx5_ib_dev *ibdev = reg->ibdev; - mutex_lock(&ibdev->data_direct_lock); + WRITE_ONCE(ibdev->data_direct_dev, NULL); blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, NULL); - ibdev->data_direct_dev = NULL; - mutex_unlock(&ibdev->data_direct_lock); } int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index d0141509870a..cefbcc816651 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3948,7 +3948,9 @@ static int mlx5_ib_data_direct_event(struct notifier_block *nb, if (action != MLX5_DATA_DIRECT_UNBIND) return NOTIFY_DONE; + mutex_lock(&dev->data_direct_lock); mlx5_ib_revoke_data_direct_mrs(dev); + mutex_unlock(&dev->data_direct_lock); return NOTIFY_OK; } diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e56f3eee7ba1..8792e10340c5 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1092,7 +1092,9 @@ struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; struct mlx5_data_direct_dev *data_direct_dev; - /* protect accessing data_direct_dev */ + /* Protects data_direct_mr_list and serializes mr + * registration/deregistration with data direct device unbind. + */ struct mutex data_direct_lock; struct notifier_block mdev_events; struct notifier_block sys_error_events; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index e6b74955d95d..1fc61a680f8a 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -997,7 +997,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, return ERR_PTR(-EOPNOTSUPP); mutex_lock(&dev->data_direct_lock); - data_direct_dev = dev->data_direct_dev; + data_direct_dev = READ_ONCE(dev->data_direct_dev); if (!data_direct_dev) { ret = -EINVAL; goto end; diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 1ee31611b4b3..df949fa634c9 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -203,7 +203,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( return PTR_ERR(c); dev = to_mdev(c->ibucontext.device); mutex_lock(&dev->data_direct_lock); - data_direct_dev = dev->data_direct_dev; + data_direct_dev = READ_ONCE(dev->data_direct_dev); if (!data_direct_dev) { ret = -ENODEV; goto end; From f6c452b311f0e4a8fc2a0c287dde298d5492112a Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:25 +0300 Subject: [PATCH 118/153] RDMA/mlx5: Consolidate data direct state into one object There are three per device data direct objects: mlx5_data_direct_dev, mlx5_data_direct_resources and mlx5_data_direct_registration with separate allocations. This patch consolidates all of them into the registration object. For this, a few things needed to happen: - mlx5_ib_dev keeps a pointer to mlx5_data_direct. - mlx5_data_direct_dev is now accessible from mlx5_data_direct. mlx5_data_direct_get_dev() is added as a helper to access the device pointer. - Since mlx5_data_direct is part of mlx5_data_direct_registration, getting to the registration is a simple container_of. No need for searching the list. Since mlx5_data_direct_resources no longer has an accurate name, this patch renames it. This is a preparation for the upcoming move of data_direct to mlx5_core. This patch has no functional changes. Signed-off-by: Dragos Tatulea Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-9-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 73 ++++++++++-------------- drivers/infiniband/hw/mlx5/data_direct.h | 7 --- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +- drivers/infiniband/hw/mlx5/mr.c | 6 +- drivers/infiniband/hw/mlx5/odp.c | 2 +- drivers/infiniband/hw/mlx5/std_types.c | 2 +- drivers/infiniband/hw/mlx5/umr.c | 11 ++-- include/linux/mlx5/data_direct.h | 17 ++++++ 8 files changed, 59 insertions(+), 62 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index 16afacdd3601..f58a144d95e3 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -18,10 +18,10 @@ static LIST_HEAD(mlx5_data_direct_reg_list); static DEFINE_MUTEX(mlx5_data_direct_mutex); struct mlx5_data_direct_registration { - struct mlx5_ib_dev *ibdev; char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; struct list_head list; struct blocking_notifier_head users; + struct mlx5_data_direct dd; }; static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, @@ -139,8 +139,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) if (err) goto err_mkey; - dev->ddr.mkey = mkey; - dev->ddr.pdn = pdn; + dev->data_direct->mkey = mkey; + dev->data_direct->pdn = pdn; /* create another mkey with RO support */ if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) { @@ -157,8 +157,8 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); /* RO is defined as best effort */ if (!err) { - dev->ddr.mkey_ro = mkey; - dev->ddr.mkey_ro_valid = true; + dev->data_direct->mkey_ro = mkey; + dev->data_direct->mkey_ro_valid = true; } } @@ -174,38 +174,25 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) { - if (dev->ddr.mkey_ro_valid) - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro); + if (dev->data_direct->mkey_ro_valid) + mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro); - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey); - mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn); + mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey); + mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn); - memset(&dev->ddr, 0, sizeof(dev->ddr)); + memset(dev->data_direct, 0, sizeof(*dev->data_direct)); } -static struct mlx5_data_direct_registration * -mlx5_data_direct_get_reg(struct mlx5_ib_dev *ibdev) -{ - struct mlx5_data_direct_registration *reg; - - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) - if (reg->ibdev == ibdev) - return reg; - return NULL; -} - -static void mlx5_data_direct_bind(struct mlx5_ib_dev *ibdev, +static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg, struct mlx5_data_direct_dev *dev) { - WRITE_ONCE(ibdev->data_direct_dev, dev); + WRITE_ONCE(reg->dd.dev, dev); } static void mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) { - struct mlx5_ib_dev *ibdev = reg->ibdev; - - WRITE_ONCE(ibdev->data_direct_dev, NULL); + WRITE_ONCE(reg->dd.dev, NULL); blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, NULL); } @@ -223,7 +210,6 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) if (!reg) return -ENOMEM; - reg->ibdev = ibdev; BLOCKING_INIT_NOTIFIER_HEAD(®->users); err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid); @@ -234,10 +220,12 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) return err; } + ibdev->data_direct = ®->dd; + mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) { - mlx5_data_direct_bind(ibdev, dev); + mlx5_data_direct_bind(reg, dev); break; } } @@ -257,14 +245,14 @@ void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) if (!mlx5_data_direct_supported(ibdev->mdev)) return; + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) { - list_del(®->list); - mlx5_data_direct_do_unbind(reg); - } + list_del(®->list); + mlx5_data_direct_do_unbind(reg); mutex_unlock(&mlx5_data_direct_mutex); + ibdev->data_direct = NULL; kfree(reg); } @@ -276,11 +264,10 @@ int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, if (!mlx5_data_direct_supported(ibdev->mdev)) return 0; - mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) - blocking_notifier_chain_register(®->users, nb); - mutex_unlock(&mlx5_data_direct_mutex); + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_register(®->users, nb); + return 0; } EXPORT_SYMBOL_GPL(mlx5_data_direct_register); @@ -293,11 +280,9 @@ void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, if (!mlx5_data_direct_supported(ibdev->mdev)) return; - mutex_lock(&mlx5_data_direct_mutex); - reg = mlx5_data_direct_get_reg(ibdev); - if (reg) - blocking_notifier_chain_unregister(®->users, nb); - mutex_unlock(&mlx5_data_direct_mutex); + reg = container_of(ibdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_unregister(®->users, nb); } EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); @@ -308,7 +293,7 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_data_direct_bind(reg->ibdev, dev); + mlx5_data_direct_bind(reg, dev); } /* Add the data direct device to the global list, further IB devices may diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index f207239461bf..de2730f00852 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -23,13 +23,6 @@ struct mlx5_data_direct_dev { struct list_head list; }; -struct mlx5_data_direct_resources { - u32 pdn; - u32 mkey; - u32 mkey_ro; - u8 mkey_ro_valid :1; -}; - int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev); void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 8792e10340c5..6952170f472b 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1091,7 +1091,7 @@ struct mlx5_macsec { struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; - struct mlx5_data_direct_dev *data_direct_dev; + struct mlx5_data_direct *data_direct; /* Protects data_direct_mr_list and serializes mr * registration/deregistration with data direct device unbind. */ @@ -1152,7 +1152,6 @@ struct mlx5_ib_dev { u16 pkey_table_len; u8 lag_ports; struct mlx5_special_mkeys mkeys; - struct mlx5_data_direct_resources ddr; #ifdef CONFIG_MLX5_MACSEC struct mlx5_macsec macsec; diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 1fc61a680f8a..5efe78cb34c1 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -587,7 +587,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, populate ? pd : dev->umrc.pd); /* In case a data direct flow, overwrite the pdn field by its internal kernel PD */ if (umem->is_dmabuf && ksm_mode) - MLX5_SET(mkc, mkc, pd, dev->ddr.pdn); + MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn); MLX5_SET(mkc, mkc, free, !populate); MLX5_SET(mkc, mkc, access_mode_1_0, access_mode); @@ -997,7 +997,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, return ERR_PTR(-EOPNOTSUPP); mutex_lock(&dev->data_direct_lock); - data_direct_dev = READ_ONCE(dev->data_direct_dev); + data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); if (!data_direct_dev) { ret = -EINVAL; goto end; @@ -1006,7 +1006,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, /* If no device's 'data direct mkey' with RO flags exists * mask it out accordingly. */ - if (!dev->ddr.mkey_ro_valid) + if (!dev->data_direct->mkey_ro_valid) access_flags &= ~IB_ACCESS_RELAXED_ORDERING; crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev, offset, length, virt_addr, fd, diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 1badec9bf527..7b9f73b76d0d 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) int ret; if (mr->data_direct) - pdn = dev->ddr.pdn; + pdn = dev->data_direct->pdn; else pdn = to_mpd(pd)->pdn; diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index df949fa634c9..3828100fdc6c 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -203,7 +203,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( return PTR_ERR(c); dev = to_mdev(c->ibucontext.device); mutex_lock(&dev->data_direct_lock); - data_direct_dev = READ_ONCE(dev->data_direct_dev); + data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); if (!data_direct_dev) { ret = -ENODEV; goto end; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 48cae5cc1c1b..1ca83d2d8a0b 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -760,12 +760,14 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, } if (dd) { + struct mlx5_data_direct *mdd = dev->data_direct; + cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter)); if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING && - dev->ddr.mkey_ro_valid) - cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro); + mdd->mkey_ro_valid) + cur_ksm->key = cpu_to_be32(mdd->mkey_ro); else - cur_ksm->key = cpu_to_be32(dev->ddr.mkey); + cur_ksm->key = cpu_to_be32(mdd->mkey); if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP)) { cur_ksm->va = 0; @@ -810,7 +812,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn, + return _mlx5r_umr_update_mr_pas(mr, flags, + mr_to_mdev(mr)->data_direct->pdn, true, start_block, nblocks); } diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h index 71f47547aebb..eaa2dc6bc607 100644 --- a/include/linux/mlx5/data_direct.h +++ b/include/linux/mlx5/data_direct.h @@ -6,10 +6,27 @@ #ifndef _MLX5_DATA_DIRECT_H #define _MLX5_DATA_DIRECT_H +#include + struct mlx5_core_dev; +struct mlx5_data_direct_dev; struct mlx5_ib_dev; struct notifier_block; +struct mlx5_data_direct { + struct mlx5_data_direct_dev *dev; + u32 pdn; + u32 mkey; + u32 mkey_ro; + u8 mkey_ro_valid :1; +}; + +static inline struct mlx5_data_direct_dev * +mlx5_data_direct_get_dev(struct mlx5_data_direct *dd) +{ + return dd ? READ_ONCE(dd->dev) : NULL; +} + int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev); void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev); From 7611034dc86cfda555a49a970ef2a8664c1b8680 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:26 +0300 Subject: [PATCH 119/153] RDMA/mlx5: Pull data_direct resource creation in init phase Since data_direct resource creation happens directly after init, there's no need to separate them. This patch moves data direct resource creation in the init phase, thus simplifying the data direct API. The cleanup path gets the same treatment. This keeps the existing IB device lifetime and prepares for moving the combined initialization and cleanup to mlx5_core load and unload. The patch has no functional changes. Signed-off-by: Dragos Tatulea Signed-off-by: Tariq Toukan Link: https://patch.msgid.link/20260930114227.139274-10-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/data_direct.c | 16 ++++++++++++---- drivers/infiniband/hw/mlx5/data_direct.h | 3 --- drivers/infiniband/hw/mlx5/main.c | 7 ------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c index f58a144d95e3..3878c4aebb2c 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/infiniband/hw/mlx5/data_direct.c @@ -103,7 +103,7 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) return 0; } -int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) +static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) { int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); struct mlx5_core_dev *mdev = dev->mdev; @@ -172,15 +172,13 @@ int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) return err; } -void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) +static void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) { if (dev->data_direct->mkey_ro_valid) mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro); mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey); mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn); - - memset(dev->data_direct, 0, sizeof(*dev->data_direct)); } static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg, @@ -222,6 +220,10 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) ibdev->data_direct = ®->dd; + err = mlx5_data_direct_create_resources(ibdev); + if (err) + goto err_resources; + mutex_lock(&mlx5_data_direct_mutex); list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { if (strcmp(dev->vuid, reg->vuid) == 0) { @@ -236,6 +238,11 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) list_add_tail(®->list, &mlx5_data_direct_reg_list); mutex_unlock(&mlx5_data_direct_mutex); return 0; + +err_resources: + ibdev->data_direct = NULL; + kfree(reg); + return err; } void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) @@ -252,6 +259,7 @@ void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) mlx5_data_direct_do_unbind(reg); mutex_unlock(&mlx5_data_direct_mutex); + mlx5_data_direct_free_resources(ibdev); ibdev->data_direct = NULL; kfree(reg); } diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h index de2730f00852..49cdd407018a 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ b/drivers/infiniband/hw/mlx5/data_direct.h @@ -23,9 +23,6 @@ struct mlx5_data_direct_dev { struct list_head list; }; -int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev); -void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev); - int mlx5_data_direct_driver_register(void); void mlx5_data_direct_driver_unregister(void); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index cefbcc816651..941928a782bd 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3966,10 +3966,6 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) if (ret) return ret; - ret = mlx5_data_direct_create_resources(dev); - if (ret) - goto err_resources; - INIT_LIST_HEAD(&dev->data_direct_mr_list); dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event; ret = mlx5_data_direct_register(dev, &dev->data_direct_nb); @@ -3979,8 +3975,6 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) return ret; err_register: - mlx5_data_direct_free_resources(dev); -err_resources: mlx5_data_direct_cleanup(dev); return ret; @@ -3992,7 +3986,6 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) return; mlx5_data_direct_unregister(dev, &dev->data_direct_nb); - mlx5_data_direct_free_resources(dev); mlx5_data_direct_cleanup(dev); } From 00290b9ff59ef47b990d26a6b1a8b2ea6dedf5e9 Mon Sep 17 00:00:00 2001 From: Dragos Tatulea Date: Wed, 30 Sep 2026 14:42:27 +0300 Subject: [PATCH 120/153] mlx5: Move data direct implementation to mlx5_core As data direct will also be used by netdev devices, move the data direct code to mlx5_core. Now mlx5_core is responsible for data direct initialization and resource creation. Now data_direct is initialized in mlx5_load() before attaching consumers and clean it up in mlx5_unload() after detaching them. The data direct PCI driver is moved and registered in mlx5_core. IB specific work stays in mlx5_ib: notifier registration, mutex initialization and MR list initialization. Failure to query the VUID is now fatal because this is done only on devices that support data direct. Other than this, this patch has no additional functional changes. Signed-off-by: Dragos Tatulea Link: https://patch.msgid.link/20260930114227.139274-11-tariqt@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/Makefile | 1 - drivers/infiniband/hw/mlx5/data_direct.h | 29 ------ drivers/infiniband/hw/mlx5/main.c | 23 +---- drivers/infiniband/hw/mlx5/mlx5_ib.h | 3 +- drivers/infiniband/hw/mlx5/mr.c | 7 +- drivers/infiniband/hw/mlx5/odp.c | 2 +- drivers/infiniband/hw/mlx5/std_types.c | 3 +- drivers/infiniband/hw/mlx5/umr.c | 4 +- .../net/ethernet/mellanox/mlx5/core/Makefile | 2 +- .../mellanox/mlx5/core}/data_direct.c | 89 ++++++++++--------- .../net/ethernet/mellanox/mlx5/core/main.c | 15 ++++ .../ethernet/mellanox/mlx5/core/mlx5_core.h | 5 ++ include/linux/mlx5/data_direct.h | 23 +++-- include/linux/mlx5/driver.h | 2 + 14 files changed, 95 insertions(+), 113 deletions(-) delete mode 100644 drivers/infiniband/hw/mlx5/data_direct.h rename drivers/{infiniband/hw/mlx5 => net/ethernet/mellanox/mlx5/core}/data_direct.c (80%) diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile index dd7bb377f491..35afc7bc57d2 100644 --- a/drivers/infiniband/hw/mlx5/Makefile +++ b/drivers/infiniband/hw/mlx5/Makefile @@ -6,7 +6,6 @@ mlx5_ib-y := ah.o \ cong.o \ counters.o \ cq.o \ - data_direct.o \ dm.o \ dmah.o \ doorbell.o \ diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h deleted file mode 100644 index 49cdd407018a..000000000000 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ /dev/null @@ -1,29 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ -/* - * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved - */ - -#ifndef _MLX5_IB_DATA_DIRECT_H -#define _MLX5_IB_DATA_DIRECT_H - -#include -#include - -struct mlx5_core_dev; -struct mlx5_ib_dev; - -enum mlx5_data_direct_event { - MLX5_DATA_DIRECT_UNBIND, -}; - -struct mlx5_data_direct_dev { - struct device *device; - struct pci_dev *pdev; - char *vuid; - struct list_head list; -}; - -int mlx5_data_direct_driver_register(void); -void mlx5_data_direct_driver_unregister(void); - -#endif diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 941928a782bd..dd2a3996b14e 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -52,7 +52,6 @@ #include #include #include "macsec.h" -#include "data_direct.h" #include "dmah.h" #define UVERBS_MODULE_NAME mlx5_ib @@ -3962,20 +3961,9 @@ static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) if (!mlx5_data_direct_supported(dev->mdev)) return 0; - ret = mlx5_data_direct_init(dev); - if (ret) - return ret; - INIT_LIST_HEAD(&dev->data_direct_mr_list); dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event; - ret = mlx5_data_direct_register(dev, &dev->data_direct_nb); - if (ret) - goto err_register; - - return ret; - -err_register: - mlx5_data_direct_cleanup(dev); + ret = mlx5_data_direct_register(dev->mdev, &dev->data_direct_nb); return ret; } @@ -3985,8 +3973,7 @@ static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) if (!mlx5_data_direct_supported(dev->mdev)) return; - mlx5_data_direct_unregister(dev, &dev->data_direct_nb); - mlx5_data_direct_cleanup(dev); + mlx5_data_direct_unregister(dev->mdev, &dev->data_direct_nb); } static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) @@ -5438,9 +5425,6 @@ static int __init mlx5_ib_init(void) ret = mlx5r_rep_init(); if (ret) goto rep_err; - ret = mlx5_data_direct_driver_register(); - if (ret) - goto dd_err; ret = auxiliary_driver_register(&mlx5r_mp_driver); if (ret) goto mp_err; @@ -5453,8 +5437,6 @@ static int __init mlx5_ib_init(void) drv_err: auxiliary_driver_unregister(&mlx5r_mp_driver); mp_err: - mlx5_data_direct_driver_unregister(); -dd_err: mlx5r_rep_cleanup(); rep_err: mlx5_ib_qp_event_cleanup(); @@ -5466,7 +5448,6 @@ static int __init mlx5_ib_init(void) static void __exit mlx5_ib_cleanup(void) { - mlx5_data_direct_driver_unregister(); auxiliary_driver_unregister(&mlx5r_driver); auxiliary_driver_unregister(&mlx5r_mp_driver); mlx5r_rep_cleanup(); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index 6952170f472b..1a3b9cb79184 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -24,7 +24,7 @@ #include #include -#include "data_direct.h" +#include #include "srq.h" #include "qp.h" #include "macsec.h" @@ -1091,7 +1091,6 @@ struct mlx5_macsec { struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; - struct mlx5_data_direct *data_direct; /* Protects data_direct_mr_list and serializes mr * registration/deregistration with data direct device unbind. */ diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 5efe78cb34c1..7463b36a36b6 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -44,7 +44,6 @@ #include "dm.h" #include "mlx5_ib.h" #include "umr.h" -#include "data_direct.h" #include "dmah.h" static int mkey_max_umr_order(struct mlx5_ib_dev *dev) @@ -587,7 +586,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, populate ? pd : dev->umrc.pd); /* In case a data direct flow, overwrite the pdn field by its internal kernel PD */ if (umem->is_dmabuf && ksm_mode) - MLX5_SET(mkc, mkc, pd, dev->data_direct->pdn); + MLX5_SET(mkc, mkc, pd, dev->mdev->data_direct->pdn); MLX5_SET(mkc, mkc, free, !populate); MLX5_SET(mkc, mkc, access_mode_1_0, access_mode); @@ -997,7 +996,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, return ERR_PTR(-EOPNOTSUPP); mutex_lock(&dev->data_direct_lock); - data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); + data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct); if (!data_direct_dev) { ret = -EINVAL; goto end; @@ -1006,7 +1005,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, /* If no device's 'data direct mkey' with RO flags exists * mask it out accordingly. */ - if (!dev->data_direct->mkey_ro_valid) + if (!dev->mdev->data_direct->mkey_ro_valid) access_flags &= ~IB_ACCESS_RELAXED_ORDERING; crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev, offset, length, virt_addr, fd, diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 7b9f73b76d0d..2e9a704acc44 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -952,7 +952,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) int ret; if (mr->data_direct) - pdn = dev->data_direct->pdn; + pdn = dev->mdev->data_direct->pdn; else pdn = to_mpd(pd)->pdn; diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 3828100fdc6c..b705b88dc917 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -10,7 +10,6 @@ #include #include #include "mlx5_ib.h" -#include "data_direct.h" #define UVERBS_MODULE_NAME mlx5_ib #include @@ -203,7 +202,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( return PTR_ERR(c); dev = to_mdev(c->ibucontext.device); mutex_lock(&dev->data_direct_lock); - data_direct_dev = mlx5_data_direct_get_dev(dev->data_direct); + data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct); if (!data_direct_dev) { ret = -ENODEV; goto end; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 1ca83d2d8a0b..a3bb8d03fd35 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -760,7 +760,7 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, } if (dd) { - struct mlx5_data_direct *mdd = dev->data_direct; + struct mlx5_data_direct *mdd = dev->mdev->data_direct; cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter)); if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING && @@ -813,7 +813,7 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, return -EINVAL; return _mlx5r_umr_update_mr_pas(mr, flags, - mr_to_mdev(mr)->data_direct->pdn, + mr_to_mdev(mr)->mdev->data_direct->pdn, true, start_block, nblocks); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 19e50f0d55af..a1871259210d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -18,7 +18,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \ diag/fw_tracer.o diag/crdump.o devlink.o sh_devlink.o diag/rsc_dump.o \ diag/reporter_vnic.o fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o \ - lib/nv_param.o + lib/nv_param.o data_direct.o # # Netdev basic diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c similarity index 80% rename from drivers/infiniband/hw/mlx5/data_direct.c rename to drivers/net/ethernet/mellanox/mlx5/core/data_direct.c index 3878c4aebb2c..5afb11a2a9f0 100644 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c @@ -3,11 +3,14 @@ * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved */ -#include "mlx5_ib.h" +#include +#include +#include +#include +#include +#include -#include - -#include "data_direct.h" +#include "mlx5_core.h" static LIST_HEAD(mlx5_data_direct_dev_list); static LIST_HEAD(mlx5_data_direct_reg_list); @@ -103,10 +106,9 @@ static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) return 0; } -static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) +static int mlx5_data_direct_create_resources(struct mlx5_core_dev *mdev) { int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); - struct mlx5_core_dev *mdev = dev->mdev; bool ro_supp = false; void *mkc; u32 mkey; @@ -139,16 +141,16 @@ static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) if (err) goto err_mkey; - dev->data_direct->mkey = mkey; - dev->data_direct->pdn = pdn; + mdev->data_direct->mkey = mkey; + mdev->data_direct->pdn = pdn; /* create another mkey with RO support */ - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) { + if (MLX5_CAP_GEN(mdev, relaxed_ordering_write)) { MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); ro_supp = true; } - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) { + if (MLX5_CAP_GEN(mdev, relaxed_ordering_read)) { MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); ro_supp = true; } @@ -157,8 +159,8 @@ static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); /* RO is defined as best effort */ if (!err) { - dev->data_direct->mkey_ro = mkey; - dev->data_direct->mkey_ro_valid = true; + mdev->data_direct->mkey_ro = mkey; + mdev->data_direct->mkey_ro_valid = true; } } @@ -172,13 +174,15 @@ static int mlx5_data_direct_create_resources(struct mlx5_ib_dev *dev) return err; } -static void mlx5_data_direct_free_resources(struct mlx5_ib_dev *dev) +static void mlx5_data_direct_free_resources(struct mlx5_core_dev *mdev) { - if (dev->data_direct->mkey_ro_valid) - mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey_ro); + struct mlx5_data_direct *data_direct = mdev->data_direct; - mlx5_core_destroy_mkey(dev->mdev, dev->data_direct->mkey); - mlx5_core_dealloc_pd(dev->mdev, dev->data_direct->pdn); + if (data_direct->mkey_ro_valid) + mlx5_core_destroy_mkey(mdev, data_direct->mkey_ro); + + mlx5_core_destroy_mkey(mdev, data_direct->mkey); + mlx5_core_dealloc_pd(mdev, data_direct->pdn); } static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg, @@ -195,13 +199,13 @@ mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) NULL); } -int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) +int mlx5_data_direct_init(struct mlx5_core_dev *mdev) { struct mlx5_data_direct_registration *reg; struct mlx5_data_direct_dev *dev; int err; - if (!mlx5_data_direct_supported(ibdev->mdev)) + if (!mlx5_data_direct_supported(mdev)) return 0; reg = kzalloc_obj(*reg); @@ -210,17 +214,16 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) BLOCKING_INIT_NOTIFIER_HEAD(®->users); - err = mlx5_data_direct_query_vuid(ibdev->mdev, reg->vuid); + err = mlx5_data_direct_query_vuid(mdev, reg->vuid); if (err) { - mlx5_ib_warn(ibdev, "Failed to query VUID, disabling data direct, err=%d\n", - err); + mlx5_core_warn(mdev, "Failed to query VUID, err=%d\n", err); kfree(reg); - return err; + return -EINVAL; } - ibdev->data_direct = ®->dd; + mdev->data_direct = ®->dd; - err = mlx5_data_direct_create_resources(ibdev); + err = mlx5_data_direct_create_resources(mdev); if (err) goto err_resources; @@ -233,63 +236,63 @@ int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev) } /* Add the registration to its global list, to be used upon bind/unbind - * of its affiliated data direct device + * of its affiliated data direct device. */ list_add_tail(®->list, &mlx5_data_direct_reg_list); mutex_unlock(&mlx5_data_direct_mutex); return 0; err_resources: - ibdev->data_direct = NULL; + mdev->data_direct = NULL; kfree(reg); return err; } -void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev) +void mlx5_data_direct_cleanup(struct mlx5_core_dev *mdev) { struct mlx5_data_direct_registration *reg; - if (!mlx5_data_direct_supported(ibdev->mdev)) + if (!mlx5_data_direct_supported(mdev)) return; - reg = container_of(ibdev->data_direct, - struct mlx5_data_direct_registration, dd); + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); mutex_lock(&mlx5_data_direct_mutex); list_del(®->list); mlx5_data_direct_do_unbind(reg); mutex_unlock(&mlx5_data_direct_mutex); - mlx5_data_direct_free_resources(ibdev); - ibdev->data_direct = NULL; + mlx5_data_direct_free_resources(mdev); + mdev->data_direct = NULL; kfree(reg); } -int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, +int mlx5_data_direct_register(struct mlx5_core_dev *mdev, struct notifier_block *nb) { struct mlx5_data_direct_registration *reg; - if (!mlx5_data_direct_supported(ibdev->mdev)) + if (!mlx5_data_direct_supported(mdev)) return 0; - reg = container_of(ibdev->data_direct, - struct mlx5_data_direct_registration, dd); + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); blocking_notifier_chain_register(®->users, nb); return 0; } EXPORT_SYMBOL_GPL(mlx5_data_direct_register); -void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, +void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev, struct notifier_block *nb) { struct mlx5_data_direct_registration *reg; - if (!mlx5_data_direct_supported(ibdev->mdev)) + if (!mlx5_data_direct_supported(mdev)) return; - reg = container_of(ibdev->data_direct, - struct mlx5_data_direct_registration, dd); + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); blocking_notifier_chain_unregister(®->users, nb); } EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); @@ -304,8 +307,8 @@ static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) mlx5_data_direct_bind(reg, dev); } - /* Add the data direct device to the global list, further IB devices may - * use it later as well + /* Add the data direct device to the global list, further mlx5 devices + * may use it later as well. */ list_add_tail(&dev->list, &mlx5_data_direct_dev_list); mutex_unlock(&mlx5_data_direct_mutex); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 643b4aac2033..34f0375b6851 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -1339,8 +1340,14 @@ static int mlx5_load(struct mlx5_core_dev *dev) if (err) goto err_traps_reg; + err = mlx5_data_direct_init(dev); + if (err) + goto err_data_direct; + return 0; +err_data_direct: + mlx5_devlink_traps_unregister(priv_to_devlink(dev)); err_traps_reg: mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); @@ -1371,6 +1378,7 @@ static int mlx5_load(struct mlx5_core_dev *dev) static void mlx5_unload(struct mlx5_core_dev *dev) { + mlx5_data_direct_cleanup(dev); mlx5_devlink_traps_unregister(priv_to_devlink(dev)); mlx5_vhca_event_stop(dev); mlx5_sf_dev_table_destroy(dev); @@ -2360,6 +2368,10 @@ static int __init mlx5_init(void) if (err) goto err_sf; + err = mlx5_data_direct_driver_register(); + if (err) + goto err_dd; + err = pci_register_driver(&mlx5_core_driver); if (err) goto err_pci; @@ -2367,6 +2379,8 @@ static int __init mlx5_init(void) return 0; err_pci: + mlx5_data_direct_driver_unregister(); +err_dd: mlx5_sf_driver_unregister(); err_sf: mlx5e_cleanup(); @@ -2378,6 +2392,7 @@ static int __init mlx5_init(void) static void __exit mlx5_cleanup(void) { pci_unregister_driver(&mlx5_core_driver); + mlx5_data_direct_driver_unregister(); mlx5_sf_driver_unregister(); mlx5e_cleanup(); mlx5_unregister_debugfs(); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 09e669f83dba..fd93530a3938 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -530,4 +530,9 @@ static inline bool mlx5_pcie_cong_event_supported(struct mlx5_core_dev *dev) return true; } + +int mlx5_data_direct_init(struct mlx5_core_dev *mdev); +void mlx5_data_direct_cleanup(struct mlx5_core_dev *mdev); +int mlx5_data_direct_driver_register(void); +void mlx5_data_direct_driver_unregister(void); #endif /* __MLX5_CORE_H__ */ diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h index eaa2dc6bc607..4cff5ff5489d 100644 --- a/include/linux/mlx5/data_direct.h +++ b/include/linux/mlx5/data_direct.h @@ -7,11 +7,23 @@ #define _MLX5_DATA_DIRECT_H #include +#include +struct device; struct mlx5_core_dev; -struct mlx5_data_direct_dev; -struct mlx5_ib_dev; struct notifier_block; +struct pci_dev; + +enum mlx5_data_direct_event { + MLX5_DATA_DIRECT_UNBIND, +}; + +struct mlx5_data_direct_dev { + struct device *device; + struct pci_dev *pdev; + char *vuid; + struct list_head list; +}; struct mlx5_data_direct { struct mlx5_data_direct_dev *dev; @@ -27,12 +39,9 @@ mlx5_data_direct_get_dev(struct mlx5_data_direct *dd) return dd ? READ_ONCE(dd->dev) : NULL; } -int mlx5_data_direct_init(struct mlx5_ib_dev *ibdev); -void mlx5_data_direct_cleanup(struct mlx5_ib_dev *ibdev); - -int mlx5_data_direct_register(struct mlx5_ib_dev *ibdev, +int mlx5_data_direct_register(struct mlx5_core_dev *mdev, struct notifier_block *nb); -void mlx5_data_direct_unregister(struct mlx5_ib_dev *ibdev, +void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev, struct notifier_block *nb); #endif diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index ee29f95ead18..ed348a82576e 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -707,6 +707,7 @@ struct mlx5_rsvd_gids { struct mlx5_clock; struct mlx5_clock_dev_state; +struct mlx5_data_direct; struct mlx5_dm; struct mlx5_fw_tracer; struct mlx5_vxlan; @@ -801,6 +802,7 @@ struct mlx5_core_dev { /* sync write combining state */ struct mutex wc_state_lock; struct devlink *shd; + struct mlx5_data_direct *data_direct; }; struct mlx5_db { From d2c08575500145dc5cb55075cd9b177626531613 Mon Sep 17 00:00:00 2001 From: Bjoern Doebel Date: Wed, 30 Sep 2026 13:55:30 +0000 Subject: [PATCH 121/153] selftests/rdma: Skip rxe_sent_rcvd_bytes when rdma_rxe is missing On a kernel built without CONFIG_RDMA_RXE the test exits 1 instead of skipping: # modprobe: FATAL: Module rdma_rxe not found in directory ... not ok 467 selftests: rdma: rxe_sent_rcvd_bytes.sh # exit=1 Every other test in this directory probes with modinfo and exits $KSFT_SKIP. Apply the same guard, identical to the one in the otherwise near-identical rxe_socket_with_netns.sh. Fixes: c67bd973695c3 ("selftest/rxe: Add selftests for perf") Assisted-by: kiro:claude-opus-5 Signed-off-by: Bjoern Doebel Link: https://patch.msgid.link/20260930135530.881281-1-doebel@amazon.de Signed-off-by: Leon Romanovsky --- tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh b/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh index 0e4fbfeebd22..89218ccfba4c 100755 --- a/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh +++ b/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh @@ -4,6 +4,8 @@ PORT=4791 MODS=("tun" "rdma_rxe") +source "$(dirname "$0")/../kselftest/ktap_helpers.sh" + exec > /dev/null # --- Helper: Cleanup Routine --- @@ -24,6 +26,10 @@ if [[ $EUID -ne 0 ]]; then fi for m in "${MODS[@]}"; do + if ! modinfo "$m" >/dev/null 2>&1; then + echo "SKIP: Kernel module '$m' not found." >&2 + exit $KSFT_SKIP + fi modprobe "$m" || { echo "Error: Failed to load $m"; exit 1; } done From 3255d6b83491574b115806ec958d8f6ca090fa9c Mon Sep 17 00:00:00 2001 From: Quanye Yang Date: Mon, 5 Oct 2026 02:01:15 -0700 Subject: [PATCH 122/153] RDMA/rtrs-clt: Fix recv repost to satisfy ib_drain_rq On disconnect the client error recovery path calls ib_drain_qp(), which requires one free RQ slot and that no recv WQEs are posted while the queue is being drained. When the server is loaded with always_invalidate=N, the path does not advertise RTRS_MSG_NEW_RKEY_F. A client-requested invalidation is then a standalone SEND_WITH_INV. The recv completion path did not repost that WQE; it posted two empty recv WQEs later, when the RDMA-with-IMM response carried RTRS_IO_RSP_W_INV_IMM. If the invalidate receive never arrived, each such response increased RQ occupancy by one until ib_post_recv() started returning -ENOMEM and __ib_drain_rq() failed with: failed to drain recv queue: -12 Repost one empty recv WQE for every receive completion, including SEND_WITH_INV completions on the always_invalidate=N path, and only while the path is still connected so CQ polling cannot race with ib_drain_qp(). Size IO connection RQs to 2 * queue_depth + 1 to match the posted recv count and leave the drain slot. Tested with a synthetic RTRS client flooding unpaired RTRS_IO_RSP_W_INV_IMM messages; error recovery no longer triggers __ib_drain_rq() warnings. Fixes: 6a98d71daea1 ("RDMA/rtrs: client: main functionality") Signed-off-by: Quanye Yang Link: https://lore.kernel.org/linux-rdma/CA+0ovCihE2_pxLimjmmR8NMz4g-ie8dvvq1jmAj-Om33y3Vzfg@mail.gmail.com Link: https://patch.msgid.link/20261005-rtrs-fix-ib-drain-rq-v2-1-35535a3ed115@proton.me Tested-by: Quanye Yang Reported-by: Farhad Alemi Acked-by: Jack Wang Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/rtrs/rtrs-clt.c | 45 +++++++++----------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index 927e3f30483c..bf3a0377da3f 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -587,25 +587,21 @@ static struct ib_cqe io_comp_cqe = { .done = rtrs_clt_rdma_done }; -/* - * Post x2 empty WRs: first is for this RDMA with IMM, - * second is for RECV with INV, which happened earlier. - */ -static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe) +static int rtrs_clt_repost_recv(struct rtrs_clt_con *con) { - struct ib_recv_wr wr_arr[2], *wr; - int i; + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); + int err; - memset(wr_arr, 0, sizeof(wr_arr)); - for (i = 0; i < ARRAY_SIZE(wr_arr); i++) { - wr = &wr_arr[i]; - wr->wr_cqe = cqe; - if (i) - /* Chain backwards */ - wr->next = &wr_arr[i - 1]; + if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) + return 0; + + err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); + if (err) { + rtrs_err(con->c.path, "rtrs_post_recv_empty(): %pe\n", + ERR_PTR(err)); + rtrs_rdma_error_recovery(con); } - - return ib_post_recv(con->qp, wr, NULL); + return err; } static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) @@ -666,19 +662,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) rtrs_wrn(con->c.path, "Unknown IMM type %u\n", imm_type); } - if (w_inval) - /* - * Post x2 empty WRs: first is for this RDMA with IMM, - * second is for RECV with INV, which happened earlier. - */ - err = rtrs_post_recv_empty_x2(&con->c, &io_comp_cqe); - else - err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); - if (err) { - rtrs_err(con->c.path, "rtrs_post_recv_empty(): %pe\n", - ERR_PTR(err)); - rtrs_rdma_error_recovery(con); - } + rtrs_clt_repost_recv(con); break; case IB_WC_RECV: /* @@ -694,6 +678,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) return rtrs_clt_rkey_rsp_done(con, wc); } + rtrs_clt_repost_recv(con); break; case IB_WC_RDMA_WRITE: /* @@ -1709,7 +1694,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) clt_path->s.dev_ref++; /* QD * (REQ + RSP + FR REGS or INVS) + drain */ max_send_wr = min(wr_limit, clt_path->queue_depth * 4 + 1); - max_recv_wr = min(wr_limit, clt_path->queue_depth * 3 + 1); + max_recv_wr = min(wr_limit, clt_path->queue_depth * 2 + 1); max_send_sge = 2; } atomic_set(&con->c.sq_wr_avail, max_send_wr); From d37e98844afea39a9614cac0de837701263c90f7 Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Tue, 6 Oct 2026 15:29:08 +0000 Subject: [PATCH 123/153] RDMA/core: Clear driver_udata before destroying objects in rdma_core When uobject creation fails while copying user output data (i.e., after the HW object has been created), it calls rdma_alloc_abort_uobject() with hw_obj_valid=true which then invokes the object's destroy callback, but passes the uverbs_attr_bundle from the original creation command. The issue is that drivers are expected to validate the udata and return an error if there's unexpected content, and data from the wrong command counts as "unexpected content", so this ends up causing the driver's destroy call to fail. A similar issue exists in the rereg_mr path when a new MR is created and the old one is destroyed. Fix this by clearing driver_udata before invoking the driver's destroy method. Fixes: 6e0954b11c05 ("RDMA/uverbs: Allow drivers to create a new HW object during rereg_mr") Fixes: 0ac8903cbbe6 ("RDMA/core: Allow the ioctl layer to abort a fully created uobject") Signed-off-by: Jacob Moroni Link: https://patch.msgid.link/20261006152908.888059-1-jmoroni@google.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/ib_core_uverbs.c | 12 ------------ drivers/infiniband/core/rdma_core.c | 2 ++ drivers/infiniband/core/rdma_core.h | 17 +++++++++++------ drivers/infiniband/core/verbs.c | 3 +-- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index 41c84ffe8c09..1482d9b27b38 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -535,18 +535,6 @@ int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) } EXPORT_SYMBOL(uverbs_destroy_def_handler); -/* - * When calling a destroy function during an error unwind we need to pass in - * the udata that is sanitized of all user arguments. Ie from the driver - * perspective it looks like no udata was passed. - */ -struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) -{ - attrs->driver_udata = (struct ib_udata){}; - return &attrs->driver_udata; -} -EXPORT_SYMBOL_NS_GPL(uverbs_get_cleared_udata, "rdma_core"); - /** * _uverbs_alloc() - Quickly allocate memory for use with a bundle * @bundle: The bundle diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index a7cbe643e33c..b32e5445601b 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -734,6 +734,7 @@ void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj * If this fails then the uobject is still completely valid (though with * a new ID) and we leak it until context close. */ + uverbs_get_cleared_udata(attrs); uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs); } EXPORT_SYMBOL_NS_GPL(rdma_assign_uobject, "rdma_core"); @@ -750,6 +751,7 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, int ret; if (hw_obj_valid) { + uverbs_get_cleared_udata(attrs); ret = uobj->uapi_object->type_class->destroy_hw( uobj, RDMA_REMOVE_ABORT, attrs); /* diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 2b91e8527287..9de14e625dd3 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -71,14 +71,19 @@ int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx); void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile); -#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) -struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs); -#else -static inline struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) +/* + * When calling a destroy function during an error unwind we need to pass in + * the udata that is sanitized of all user arguments. Ie from the driver + * perspective it looks like no udata was passed. + */ +static inline struct ib_udata * +uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) { - return NULL; + if (!attrs) + return NULL; + attrs->driver_udata = (struct ib_udata){}; + return &attrs->driver_udata; } -#endif /* * This is the runtime description of the uverbs API, used by the syscall diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index 6d5825114c19..edf62652f9d3 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1331,8 +1331,7 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, return qp; err_security: - qp->device->ops.destroy_qp( - qp, uattrs ? uverbs_get_cleared_udata(uattrs) : NULL); + qp->device->ops.destroy_qp(qp, uverbs_get_cleared_udata(uattrs)); err_create: rdma_restrack_put(&qp->res); xa_destroy(&qp->comp_cntrs); From aea64024ac37d29e99b82b315a56a9145dc0bc23 Mon Sep 17 00:00:00 2001 From: Prathamesh Deshpande Date: Thu, 8 Oct 2026 00:42:11 +0100 Subject: [PATCH 124/153] RDMA/mlx5: Cancel deferred page-fault EQ work before teardown Page-fault EQ work can remain queued after the notifier is disabled. Destroying the EQ before cancelling that work can let the worker access freed EQ memory. Cancel the deferred work before destroying the EQ. Fixes: d5d284b829a6 ("{net,IB}/mlx5: Move Page fault EQ and ODP logic to RDMA") Signed-off-by: Prathamesh Deshpande Link: https://patch.msgid.link/20261007234216.2765-1-prathameshdeshpande7@gmail.com Reviewed-by: Chenguang Zhao Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/odp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 2fe0b6bc4348..9c91e76029d4 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -1885,8 +1885,8 @@ mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) if (!eq->core) return 0; mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); - err = mlx5_eq_destroy_generic(dev->mdev, eq->core); cancel_work_sync(&eq->work); + err = mlx5_eq_destroy_generic(dev->mdev, eq->core); destroy_workqueue(eq->wq); mempool_destroy(eq->pool); From a0ab2bb269e7fd8523f87950b91a42a36c7f8f2a Mon Sep 17 00:00:00 2001 From: Prathamesh Deshpande Date: Thu, 8 Oct 2026 15:12:02 +0100 Subject: [PATCH 125/153] RDMA/mlx5: Cancel software CQ notifications before teardown GSI completions queue notify_work to invoke the CQ completion handler. CQ destruction drains hardware events and core polling, but does not cancel this software notifier. Fixes: 25361e02c448 ("IB/mlx5: Generate completions in software") Signed-off-by: Prathamesh Deshpande Link: https://patch.msgid.link/20261008141209.6923-1-prathameshdeshpande7@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/cq.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 072b77a2c8a6..7495e20c3dec 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -1117,6 +1117,9 @@ int mlx5_ib_pre_destroy_cq(struct ib_cq *cq) struct mlx5_ib_dev *dev = to_mdev(cq->device); struct mlx5_ib_cq *mcq = to_mcq(cq); + if (!cq->uobject) + cancel_work_sync(&mcq->notify_work); + return mlx5_core_destroy_cq(dev->mdev, &mcq->mcq); } From f74eac4fc6c56274fb96c2c44ffad8f01c325f17 Mon Sep 17 00:00:00 2001 From: Haitam Lazaar Date: Sun, 11 Oct 2026 11:55:57 +0000 Subject: [PATCH 126/153] RDMA/siw: Enforce IB_ACCESS_REMOTE_READ for read responses When generating an RDMA READ response (SIW_OP_READ_RESPONSE), siw_check_sgl_tx() unconditionally passes 0 as the required permission mask: /* Reference memory to be tx'd w/o checking access for LOCAL_READ */ rv = siw_check_sgl_tx(qp->pd, wqe, 0); Because perms is 0, siw_check_mem()'s authorization test: if ((mem->perms & perms) < perms) evaluates to (mem->perms & 0) < 0, which is always false. Consequently, an untrusted remote peer can issue an RDMA Read targeting any valid STag in the Protection Domain, even if the memory was registered strictly as write-only (IB_ACCESS_REMOTE_WRITE) without IB_ACCESS_REMOTE_READ. Enforce that the target memory region was granted IB_ACCESS_REMOTE_READ when answering SIW_OP_READ_RESPONSE work queue entries, while preserving perms = 0 for standard local sends. Fixes: b9be6f18cf9e ("rdma/siw: transmit path") Signed-off-by: Haitam Lazaar Link: https://patch.msgid.link/1a12ad1d83f.19cd98924357428.7073029566232810804@lazaarsec.com Acked-by: Bernard Metzler Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/siw/siw_qp_tx.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c index 69fdcfffea80..2675bc0b2019 100644 --- a/drivers/infiniband/sw/siw/siw_qp_tx.c +++ b/drivers/infiniband/sw/siw/siw_qp_tx.c @@ -804,17 +804,21 @@ static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe) if (wqe->wr_status == SIW_WR_QUEUED) { if (!(wqe->sqe.flags & SIW_WQE_INLINE)) { - if (tx_type(wqe) == SIW_OP_READ_RESPONSE) - wqe->sqe.num_sge = 1; - if (tx_type(wqe) != SIW_OP_READ && tx_type(wqe) != SIW_OP_READ_LOCAL_INV) { + enum ib_access_flags perms = 0; + /* - * Reference memory to be tx'd w/o checking - * access for LOCAL_READ permission, since - * not defined in RDMA core. + * Local READ permission doesn't exist in RDMA core. + * A READ RESPONSE source is read by the remote peer, + * so it must be checked for REMOTE_READ permission. */ - rv = siw_check_sgl_tx(qp->pd, wqe, 0); + if (tx_type(wqe) == SIW_OP_READ_RESPONSE) { + perms = IB_ACCESS_REMOTE_READ; + wqe->sqe.num_sge = 1; + } + + rv = siw_check_sgl_tx(qp->pd, wqe, perms); if (rv < 0) { if (tx_type(wqe) == SIW_OP_READ_RESPONSE) From 466612f8a8020c6b3af6377c730775a4c630f12c Mon Sep 17 00:00:00 2001 From: Yonatan Nachum Date: Mon, 12 Oct 2026 16:51:13 +0000 Subject: [PATCH 127/153] RDMA/efa: Fix race between admin command timeout and completion An admin command that times out can race with the interrupt handler processing its completion, in two ways. The timeout path releases the completion context without synchronizing against the handler. A handler that has already seen the command as submitted then copies the CQE into the caller's response buffer after the waiter returned and that buffer went out of scope. The waiter can also observe the command as completed before the CQE was copied, since the status is published first. It decodes its own uninitialized response buffer, returns success with garbage contents and leaks stack data to the caller. Both wait paths read the status without the CQ lock. Add a timed-out state and publish it under the CQ lock in both wait paths before returning, so the handler, which only completes commands in submitted state, cannot act on an abandoned context. In the interrupt wait path test the return value of the wait instead of the status: the handler signals the completion after copying the CQE, so a successful wait implies a valid response. Read the status in the polling wait path inside the critical section that drains the CQ, which also lets a completion arriving on the last iteration be honored instead of reported as a timeout. Fixes: 0420e542569b ("RDMA/efa: Implement functions that submit and complete admin commands") Reviewed-by: Michael Margolin Reviewed-by: Tom Sela Signed-off-by: Yonatan Nachum Link: https://patch.msgid.link/20261012165113.576899-1-ynachum@amazon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/efa/efa_com.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index 04c6d63c449e..9c59d127e1e0 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -41,6 +41,7 @@ enum efa_cmd_status { EFA_CMD_ALLOCATED, EFA_CMD_SUBMITTED, EFA_CMD_COMPLETED, + EFA_CMD_TIMED_OUT, }; struct efa_comp_ctx { @@ -591,6 +592,7 @@ static int efa_com_comp_status_to_errno(u8 comp_status) static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx, struct efa_com_admin_queue *aq) { + bool comp_timeout = false; unsigned long timeout; unsigned long flags; @@ -599,12 +601,19 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c while (1) { spin_lock_irqsave(&aq->cq.lock, flags); efa_com_handle_admin_completion(aq); - spin_unlock_irqrestore(&aq->cq.lock, flags); - - if (comp_ctx->status != EFA_CMD_SUBMITTED) + if (comp_ctx->status != EFA_CMD_SUBMITTED) { + spin_unlock_irqrestore(&aq->cq.lock, flags); break; + } if (time_is_before_jiffies(timeout)) { + comp_ctx->status = EFA_CMD_TIMED_OUT; + comp_timeout = true; + } + + spin_unlock_irqrestore(&aq->cq.lock, flags); + + if (comp_timeout) { ibdev_err_ratelimited( aq->efa_dev, "Wait for completion (polling) timeout\n"); @@ -625,10 +634,11 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx, struct efa_com_admin_queue *aq) { - unsigned long flags; + unsigned long flags, remaining_jiffies; + bool completed; - wait_for_completion_timeout(&comp_ctx->wait_event, - usecs_to_jiffies(aq->completion_timeout)); + remaining_jiffies = wait_for_completion_timeout( + &comp_ctx->wait_event, usecs_to_jiffies(aq->completion_timeout)); /* * In case the command wasn't completed find out the root cause. @@ -636,14 +646,16 @@ static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *com * 1) No completion (timeout reached) * 2) There is completion but the device didn't get any msi-x interrupt. */ - if (comp_ctx->status == EFA_CMD_SUBMITTED) { + if (remaining_jiffies == 0) { spin_lock_irqsave(&aq->cq.lock, flags); efa_com_handle_admin_completion(aq); + completed = (comp_ctx->status == EFA_CMD_COMPLETED); + comp_ctx->status = EFA_CMD_TIMED_OUT; spin_unlock_irqrestore(&aq->cq.lock, flags); atomic64_inc(&aq->stats.no_completion); - if (comp_ctx->status == EFA_CMD_COMPLETED) + if (completed) ibdev_err_ratelimited( aq->efa_dev, "The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n", From 724a39b36af94787d32094da87729f24a28888b7 Mon Sep 17 00:00:00 2001 From: Abhishek Mohapatra Date: Tue, 13 Oct 2026 07:04:26 -0700 Subject: [PATCH 128/153] RDMA/bnxt_re: Use kvzalloc for QP table allocation The QP table size depends on the firmware reported max_qp, so it can be large and a physically contiguous kzalloc may fail. Use kvzalloc_objs() and free it with kvfree(). Fixes: 82f1f575aa13 ("RDMA/bnxt_re: Fix allocation of QP table") Signed-off-by: Abhishek Mohapatra Signed-off-by: Selvin Xavier Link: https://patch.msgid.link/20261013140427.256612-2-selvin.xavier@broadcom.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/qplib_res.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 7ff587ce9126..e8a111646fc9 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -910,7 +910,7 @@ int bnxt_qplib_init_res(struct bnxt_qplib_res *res) void bnxt_qplib_free_res(struct bnxt_qplib_res *res) { - kfree(res->rcfw->qp_tbl); + kvfree(res->rcfw->qp_tbl); bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl); bnxt_qplib_free_pd_tbl(&res->pd_tbl); bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl); @@ -927,8 +927,8 @@ int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct net_device *netdev) /* Allocate one extra to hold the QP1 entries */ rcfw->qp_tbl_size = max_t(u32, BNXT_RE_MAX_QPC_COUNT + 1, dev_attr->max_qp); - rcfw->qp_tbl = kzalloc_objs(struct bnxt_qplib_qp_node, - rcfw->qp_tbl_size); + rcfw->qp_tbl = kvzalloc_objs(struct bnxt_qplib_qp_node, + rcfw->qp_tbl_size); if (!rcfw->qp_tbl) return -ENOMEM; From 978361dd2ccff357d4fe6aac6ce057d1bfae8312 Mon Sep 17 00:00:00 2001 From: Selvin Xavier Date: Tue, 13 Oct 2026 07:04:27 -0700 Subject: [PATCH 129/153] RDMA/bnxt_re: Unpublish QP table before freeing it bnxt_re_dev_uninit() frees the QP table before the RCFW channel is disabled, so the CREQ tasklet can still look up rcfw->qp_tbl while it is being freed. Clear rcfw->qp_tbl under tbl_lock before freeing it, and have the QP error notification handler bail out if the table is gone. Fixes: 82f1f575aa13 ("RDMA/bnxt_re: Fix allocation of QP table") Signed-off-by: Selvin Xavier Link: https://patch.msgid.link/20261013140427.256612-3-selvin.xavier@broadcom.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/bnxt_re/qplib_rcfw.c | 4 ++++ drivers/infiniband/hw/bnxt_re/qplib_res.c | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c index 9ee0d500436c..34d8aa12265d 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -638,6 +638,10 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, err_event = (struct creq_qp_error_notification *)qp_event; qp_id = le32_to_cpu(err_event->xid); spin_lock(&rcfw->tbl_lock); + if (!rcfw->qp_tbl) { + spin_unlock(&rcfw->tbl_lock); + break; + } tbl_indx = map_qp_id_to_tbl_indx(qp_id, rcfw); qp = rcfw->qp_tbl[tbl_indx].qp_handle; if (!qp) { diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index e8a111646fc9..cde56ff2313b 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -910,7 +910,15 @@ int bnxt_qplib_init_res(struct bnxt_qplib_res *res) void bnxt_qplib_free_res(struct bnxt_qplib_res *res) { - kvfree(res->rcfw->qp_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_qp_node *qp_tbl; + + /* The CREQ tasklet may still be running; unpublish the table first */ + spin_lock_bh(&rcfw->tbl_lock); + qp_tbl = rcfw->qp_tbl; + rcfw->qp_tbl = NULL; + spin_unlock_bh(&rcfw->tbl_lock); + kvfree(qp_tbl); bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl); bnxt_qplib_free_pd_tbl(&res->pd_tbl); bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl); From 66196a559be5963b1ced18418c876caf9bfb76e6 Mon Sep 17 00:00:00 2001 From: Carolina Jubran Date: Wed, 14 Oct 2026 13:47:04 +0300 Subject: [PATCH 130/153] IB/IPoIB: Cancel normal flush work on device stop ipoib_ib_dev_flush_normal() can race with ipoib_stop() because the normal flush work does not run under RTNL. Before commit a77a57a1a22a ("IPoIB: Fix deadlock on RTNL in ipoib_stop()"), ipoib_stop() flushed ipoib_workqueue after clearing IPOIB_FLAG_ADMIN_UP and before calling ipoib_ib_dev_down(). This ensured that any in-flight flush work completed before ipoib_ib_dev_down() performed the final clear of IPOIB_FLAG_OPER_UP. After that commit removed the workqueue flush to avoid a deadlock with workqueue tasks taking RTNL, a normal flush can continue running concurrently with ipoib_stop() after IPOIB_FLAG_ADMIN_UP has been cleared. The flush may observe IPOIB_FLAG_ADMIN_UP set and continue running while ipoib_stop() clears IPOIB_FLAG_ADMIN_UP and brings the device down. The flush can then call ipoib_ib_dev_up() and set IPOIB_FLAG_OPER_UP again after the device has already been stopped: flush_normal (ipoib_workqueue) ipoib_stop() (RTNL held) ------------------------------ ------------------------ test_bit(ADMIN_UP) == 1 clear_bit(ADMIN_UP) ipoib_ib_dev_down() clear_bit(OPER_UP) ipoib_ib_dev_up() set_bit(OPER_UP) This leaves the interface with ADMIN_UP clear and OPER_UP set. During device removal, ipoib_ndo_uninit() waits in destroy_workqueue() while holding RTNL, while ipoib_mcast_carrier_on_task() may keep retrying rtnl_trylock() as long as OPER_UP remains set, resulting in a deadlock. Cancel the normal flush work from ipoib_stop() after clearing IPOIB_FLAG_ADMIN_UP and before bringing the device down. A pending normal flush is canceled, while an already running flush is allowed to finish before ipoib_ib_dev_down() performs the final transition to OPER_UP clear. Only the normal flush needs this synchronization. The heavy flush is already serialized with ipoib_stop() by RTNL, while the light flush does not bring the device back up and therefore cannot recreate this race. Fixes: a77a57a1a22a ("IPoIB: Fix deadlock on RTNL in ipoib_stop()") Reported-by: John Popplewell Signed-off-by: Carolina Jubran Reviewed-by: Cosmin Ratiu Signed-off-by: Edward Srouji Link: https://patch.msgid.link/20261014-ipoib-cancel-work-v1-1-6c0faac016b8@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/ipoib/ipoib_main.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 6c14246befb1..f0b72d65fdeb 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -230,6 +230,13 @@ static int ipoib_stop(struct net_device *dev) clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags); + /* Normal flush runs without RTNL and may bring the device back + * up. Cancel it before the device down transition. + * Heavy flush is already serialized by RTNL and must not be + * waited on here while RTNL is held. + */ + cancel_work_sync(&priv->flush_normal); + netif_stop_queue(dev); ipoib_ib_dev_down(dev); From 9e61a2ac6590e9c42d2eeef687749781334b300f Mon Sep 17 00:00:00 2001 From: Youngsung Ahn Date: Thu, 1 Oct 2026 03:22:23 +0900 Subject: [PATCH 131/153] RDMA/rxe: refuse to destroy a QP with type 2 MWs still bound Binding a type 2 memory window takes a reference on the QP: rxe_do_bind_mw() does rxe_get(qp) and stores it in mw->qp. That reference is released only when the window is invalidated (rxe_do_invalidate_mw() on IB_WR_LOCAL_INV) or the window itself is destroyed (rxe_mw_cleanup()). rxe_qp_chk_destroy() does not account for those references -- it only refuses when qp->mcg_num is non-zero. So an unprivileged user can bind a type 2 MW to a QP and then destroy the QP while the binding is still live. __rxe_cleanup() cannot drop the QP refcount to zero (the MW still holds a reference), hits its -ETIMEDOUT path, and frees the QP anyway; mw->qp is left dangling. When the MW is later invalidated or deallocated, rxe_mw_cleanup()/rxe_do_invalidate_mw() do rxe_put(mw->qp) on the freed struct rxe_qp -- a use-after-free write (refcount_dec, and complete()/list work if it reaches zero) reachable from userspace. IB_WR_BIND_MW is a local operation, so no peer is required. Track the number of type 2 MWs bound to a QP and refuse to destroy the QP while that count is non-zero, returning -EBUSY exactly as the existing multicast-attachment check does. The window must be invalidated or deallocated, which drops the reference and the count, before the QP can be destroyed. Fixes: 8700e3e7c485 ("Soft RoCE driver") Signed-off-by: Youngsung Ahn Link: https://patch.msgid.link/20260930182223.2511145-1-ays511.kr@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_mw.c | 3 +++ drivers/infiniband/sw/rxe/rxe_qp.c | 10 ++++++++++ drivers/infiniband/sw/rxe/rxe_verbs.h | 1 + 3 files changed, 14 insertions(+) diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c index bddb7a257831..a2e3a6ce0946 100644 --- a/drivers/infiniband/sw/rxe/rxe_mw.c +++ b/drivers/infiniband/sw/rxe/rxe_mw.c @@ -161,6 +161,7 @@ static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, if (mw->ibmw.type == IB_MW_TYPE_2) { rxe_get(qp); + atomic_inc(&qp->mw_bind_num); mw->qp = qp; } } @@ -245,6 +246,7 @@ static void rxe_do_invalidate_mw(struct rxe_mw *mw) /* valid type 2 MW will always have a QP pointer */ qp = mw->qp; mw->qp = NULL; + atomic_dec(&qp->mw_bind_num); rxe_put(qp); /* valid type 2 MW will always have an MR pointer */ @@ -332,6 +334,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem) struct rxe_qp *qp = mw->qp; mw->qp = NULL; + atomic_dec(&qp->mw_bind_num); rxe_put(qp); } diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index 311f285d78a6..fea55bc3b0cb 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -858,6 +858,16 @@ int rxe_qp_chk_destroy(struct rxe_qp *qp) return -EBUSY; } + /* An attempt to destroy a QP while it still holds references for + * bound type 2 memory windows will fail immediately. Otherwise the + * QP is freed with those references outstanding and the windows are + * left pointing at freed memory (use-after-free). + */ + if (atomic_read(&qp->mw_bind_num)) { + rxe_dbg_qp(qp, "Attempt to destroy while type 2 MWs are bound\n"); + return -EBUSY; + } + return 0; } diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index 0f5ffd94643f..80df376e61ee 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -261,6 +261,7 @@ struct rxe_qp { struct rxe_av alt_av; atomic_t mcg_num; + atomic_t mw_bind_num; struct sk_buff_head req_pkts; struct sk_buff_head resp_pkts; From 33afe9909f213d8bb8e80f0b93382dde0f56e2c7 Mon Sep 17 00:00:00 2001 From: Youngsung Ahn Date: Fri, 2 Oct 2026 00:24:26 +0900 Subject: [PATCH 132/153] RDMA/rxe: reject IB_WR_REG_MR from user QPs in rxe_reg_fast_mr() IB_WR_REG_MR is documented as kernel-only ("can not be issued by userspace"), but nothing enforces it. rxe_wr_opcode_info[IB_WR_REG_MR] sets WR_LOCAL_OP_MASK for IB_QPT_RC, a QP type userspace can create, so a user QP that writes opcode = IB_WR_REG_MR into its mmap'd SQ ring reaches rxe_do_local_ops() -> rxe_reg_fast_mr(). That function then trusts the ring-supplied fields: struct rxe_mr *mr = to_rmr(wqe->wr.wr.reg.mr); ... if (unlikely(mr->state != RXE_MR_STATE_FREE)) /* deref */ ... mr->access = access; /* write */ For a user QP wqe->wr.wr.reg.mr is a fully attacker-chosen pointer, so this is an arbitrary-kernel-address dereference (arbitrary read confirmed, oops). No existing check distinguishes a user QP on this path. Reject IB_WR_REG_MR for user QPs before dereferencing the supplied pointer. Fixes: 8700e3e7c485 ("Soft RoCE driver") Signed-off-by: Youngsung Ahn Link: https://patch.msgid.link/20261001152426.3259522-1-ays511.kr@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_mr.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index 615da4bb9a38..af6e0f76b46d 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -777,6 +777,10 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe) u32 key = wqe->wr.wr.reg.key; u32 access = wqe->wr.wr.reg.access; + /* IB_WR_REG_MR is a kernel-only op; a user QP must not reach here. */ + if (qp->is_user) + return -EOPNOTSUPP; + /* user can only register MR in free state */ if (unlikely(mr->state != RXE_MR_STATE_FREE)) { rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n", mr->lkey); From 7797bbb0e86f0062fe58480a0116c86e4ec0d0d1 Mon Sep 17 00:00:00 2001 From: Youngsung Ahn Date: Mon, 5 Oct 2026 00:32:46 +0900 Subject: [PATCH 133/153] RDMA/rxe: bound the ODP page index against the umem in rxe_check_pagefault() An ODP MR has a page array, map.pfn_list[]. Its size is computed from the umem start address. But rxe_check_pagefault() builds the index into that array from an iova that is relative to the same start, while the access itself was range-checked against the MR's iova (ibmr.iova). Those two bases do not have to match. reg_mr only requires that start and hca_va have the same page offset, so a user can register an MR with hca_va = start + N*PAGE. The difference then becomes an index skew of (hca_va - start) >> PAGE_SHIFT. Nothing compares the result with the size of pfn_list[], so the read goes past the end of the array. An unprivileged local user can reach this with a single RDMA operation on a self-connected rxe QP. KASAN reports a slab-out-of-bounds read. If the out-of-bounds qword happens to look like a writable HMM pfn, rxe then copies the payload into a page the user never registered. Reject an index that falls outside the umem and take the fault path instead. That path rejects the out-of-range access in ib_umem_odp_map_dma_and_lock(), which does check the umem range. The other users of pfn_list[], __rxe_odp_mr_copy() and the atomic helpers, run only after a successful map, so this single check covers them too. Fixes: 2fae67ab63db ("RDMA/rxe: Add support for Send/Recv/Write/Read with ODP") Signed-off-by: Youngsung Ahn Link: https://patch.msgid.link/20261004153246.2951263-1-ays511.kr@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_odp.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c index c375d3efd999..9d27244f89bc 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -136,6 +136,11 @@ static inline bool rxe_check_pagefault(struct ib_umem_odp *umem_odp, u64 iova, while (addr < iova + length) { idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; + if (idx >= ib_umem_odp_num_pages(umem_odp)) { + need_fault = true; + break; + } + if ((umem_odp->map.pfn_list[idx] & access) != access) { need_fault = true; break; From 47a9c0338f2e0d4178d77b0d2f529fe1fde84642 Mon Sep 17 00:00:00 2001 From: Youngsung Ahn Date: Fri, 2 Oct 2026 00:24:33 +0900 Subject: [PATCH 134/153] RDMA/rxe: validate inline sge_offset in finish_packet() before indexing inline_data[] For a user QP, rxe_post_send() takes the qp->is_user branch and never runs init_send_wqe(), so dma.length/resid/num_sge/cur_sge/sge_offset all come straight from the mmap'd SQ ring. In the inline send path finish_packet() uses sge_offset directly as an index: u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset]; memcpy(payload_addr(pkt), tmp, payload); inline_data is a flex array whose capacity is qp->sq.max_inline, but sge_offset is a u32 from user space, so the source pointer is displaced arbitrarily past the WQE and the memcpy reads out of bounds. commit 126c757e4cd4 ("RDMA/rxe: Validate num_sge/cur_sge before indexing wqe->dma.sge[]") added num_sge/cur_sge checks on the send path but not sge_offset, the third attacker-controlled index in the same struct. Read sge_offset once with READ_ONCE() and bound it (and the payload that follows it) by qp->sq.max_inline before indexing, so the value that is validated is the value that is used. Fixes: 8700e3e7c485 ("Soft RoCE driver") Signed-off-by: Youngsung Ahn Link: https://patch.msgid.link/20261001152433.3259573-1-ays511.kr@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_req.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 53f7c1e7fe55..4f353abd76f5 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -510,7 +510,14 @@ static int finish_packet(struct rxe_qp *qp, struct rxe_av *av, if (pkt->mask & RXE_WRITE_OR_SEND_MASK) { if (wqe->wr.send_flags & IB_SEND_INLINE) { - u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset]; + u32 sge_offset = READ_ONCE(wqe->dma.sge_offset); + u8 *tmp; + + if (unlikely(sge_offset > (u32)qp->sq.max_inline || + payload > (u32)qp->sq.max_inline - sge_offset)) + return -EINVAL; + + tmp = &wqe->dma.inline_data[sge_offset]; memcpy(payload_addr(pkt), tmp, payload); From 56a27df6cedd4f3918969d61c7b9d3cc177b2d1b Mon Sep 17 00:00:00 2001 From: Honggang LI Date: Fri, 2 Oct 2026 22:22:50 +0800 Subject: [PATCH 135/153] RDMA/rxe: Fix out-of-range unsigned-to-signed conversion for RDMA message in 2GiB size When RDMA READ request for 2GiB in single WR, res->read.resid is u32 0x80000000, which is INT_MIN (-2147483648). payload = min_t(int, res->read.resid, mtu); The `min_t` function will return -2147483648 for payload. The wrong size is propagated through call chain, `read_reply` -> `prepare_ack_packet` -> `rxe_init_packet` -> `alloc_skb` . `alloc_skb` failed because of invalid size. The passive side failed to emit response packet for RDMA READ request. After fixed it, the active side of RDMA READ failed with error code "IB_WC_LOC_PROT_ERR". When RDMA_READ_RESPONSE_FIRST packet recived by the active side, `do_read` call `copy_data`. dma->resid is u32 0x80000000. int resid = dma->resid; This conversion set resid to -2147483648. `copy_data` abort as length greater than resid. RDMA SEND and WRITE 2GiB message works too, after fixed these two bugs. Fixes: 8700e3e7c485 ("Soft RoCE driver") Signed-off-by: Honggang LI Link: https://patch.msgid.link/20261002142250.657151-1-honggangli@163.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_loc.h | 4 ++-- drivers/infiniband/sw/rxe/rxe_mr.c | 8 ++++---- drivers/infiniband/sw/rxe/rxe_net.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 17 +++++++++-------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 7c3cc48e845c..e57861a62823 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -64,7 +64,7 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length); int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, unsigned int length, enum rxe_mr_copy_dir dir); int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma, - void *addr, int length, enum rxe_mr_copy_dir dir); + void *addr, u32 length, enum rxe_mr_copy_dir dir); int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, @@ -91,7 +91,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem); /* rxe_net.c */ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, - int paylen, struct rxe_pkt_info *pkt); + u32 paylen, struct rxe_pkt_info *pkt); void rxe_put_skb(struct sk_buff *skb); int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt, struct sk_buff *skb); diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index af6e0f76b46d..13cb28d3916f 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -418,13 +418,13 @@ int copy_data( int access, struct rxe_dma_info *dma, void *addr, - int length, + u32 length, enum rxe_mr_copy_dir dir) { - int bytes; + u32 bytes; struct rxe_sge *sge = &dma->sge[dma->cur_sge]; - int offset = dma->sge_offset; - int resid = dma->resid; + u32 offset = dma->sge_offset; + u32 resid = dma->resid; struct rxe_mr *mr = NULL; u64 iova; int err; diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 9f1c7f21d88d..c572cda8fe18 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -571,7 +571,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, } struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, - int paylen, struct rxe_pkt_info *pkt) + u32 paylen, struct rxe_pkt_info *pkt) { unsigned int hdr_len; struct sk_buff *skb = NULL; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index ba3b2d23b36b..1b7f31737225 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -614,7 +614,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp, } static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr, - int data_len) + u32 data_len) { int err; @@ -829,20 +829,21 @@ static enum resp_states atomic_write_reply(struct rxe_qp *qp, static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, struct rxe_pkt_info *ack, int opcode, - int payload, + u32 payload, u32 psn, u8 syndrome) { struct rxe_dev *rxe = to_rdev(qp->ibqp.device); struct sk_buff *skb; - int paylen; - int pad; + u32 paylen; + u32 pad = 0; int err; /* * allocate packet */ - pad = (-payload) & 0x3; + if (payload % 4) + pad = 4 - payload % 4; paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE; skb = rxe_init_packet(rxe, &qp->pri_av, paylen, ack); @@ -936,9 +937,9 @@ static enum resp_states read_reply(struct rxe_qp *qp, { struct rxe_pkt_info ack_pkt; struct sk_buff *skb; - int mtu = qp->mtu; + u32 mtu = qp->mtu; enum resp_states state; - int payload; + u32 payload; int opcode; int err; struct resp_res *res = qp->resp.res; @@ -984,7 +985,7 @@ static enum resp_states read_reply(struct rxe_qp *qp, res->state = rdatm_res_state_next; - payload = min_t(int, res->read.resid, mtu); + payload = min(res->read.resid, mtu); skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload, res->cur_psn, AETH_ACK_UNLIMITED); From c887e53e1c852b88bd5442d1bf4717ee3e7aadda Mon Sep 17 00:00:00 2001 From: Youngsung Ahn Date: Sat, 3 Oct 2026 01:45:52 +0900 Subject: [PATCH 136/153] RDMA/rxe: bound the WQE opcode before indexing rxe_wr_opcode_info[] wr_opcode_mask() indexes the global rxe_wr_opcode_info[] array with a caller-supplied opcode and no bounds check. For a user QP the WQE comes from an mmap'd SQ ring, so wqe->wr.opcode is an attacker-controlled __u32 read back by the requester (req_next_wqe() -> rxe_requester()); rxe_post_send() takes the qp->is_user branch and never runs validate_send_wr(), whose only opcode check is itself !wr_opcode_mask() (it indexes before it checks). rxe_wr_opcode_info[] has entries only up to IB_WR_REG_MR, so an out-of-range opcode reads out of bounds and the result is used as a mask and dereferenced; observed as a KASAN global-out-of-bounds "Read of size 4" and a wild-pointer oops. Return a zero mask for an out-of-range opcode, matching how validate_send_wr() already treats a zero mask (an invalid WR), so no path indexes the array out of bounds. Fixes: 8700e3e7c485 ("Soft RoCE driver") Signed-off-by: Youngsung Ahn Link: https://patch.msgid.link/20261002164552.294749-1-ays511.kr@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_loc.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index e57861a62823..9a8e96dfb792 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -185,6 +185,9 @@ void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb); static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp) { + if (unlikely(opcode < 0 || opcode > IB_WR_REG_MR)) + return 0; + return rxe_wr_opcode_info[opcode].mask[qp->ibqp.qp_type]; } From 9bcf45bfcaf83aed81b5fb12f0edf4c16e73e02e Mon Sep 17 00:00:00 2001 From: Praveen Kumar Kannoju Date: Wed, 14 Oct 2026 12:13:44 +0000 Subject: [PATCH 137/153] IB/mlx4: Avoid requeueing executing CM timeout work id->scheduled_delete is set by schedule_delayed() itself. It is used in two cases after the RTU timeout support was added: * A new id_map_entry is created, and a 60-second RTU timeout is scheduled. * DREQ or REJ is received for an existing entry, and the normal 30-second teardown timeout is scheduled or adjusted. The following race is possible when the RTU timeout expires while a DREQ or REJ is handled: CPU 0: timeout work CPU 1: DREQ/REJ handler -------------------- ----------------------- RTU timer expires timeout callback starts waits for id_map_lock takes id_map_lock calls schedule_delayed() calls mod_delayed_work() drops id_map_lock takes id_map_lock removes mapping frees id_map_entry requeued timeout runs accesses freed id_map_entry mod_delayed_work() may requeue the work after the timeout callback has started. The original timeout callback then frees the id_map_entry, leaving the requeued work with a dangling delayed_work pointer. When a timeout is already scheduled, first cancel its pending work. A successful cancellation means the callback has not started and the timeout can safely be scheduled again. If cancellation fails, the callback is executing and will free the entry after id_map_lock is released; do not requeue it. cancel_delayed_work() is deliberately used instead of cancel_delayed_work_sync(), because the timeout callback may be waiting for id_map_lock, which is held by the caller. Fixes: fa09adf4231f ("IB/mlx4: Fix stale CM id_map entries when RTU is never received") Signed-off-by: Praveen Kumar Kannoju Link: https://patch.msgid.link/20261014121344.100048-1-praveen.kannoju@oracle.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx4/cm.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c index 1e5e525fea73..810f95e55fda 100644 --- a/drivers/infiniband/hw/mlx4/cm.c +++ b/drivers/infiniband/hw/mlx4/cm.c @@ -301,10 +301,20 @@ static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id, spin_lock_irqsave(&sriov->going_down_lock, flags); /*make sure that there is no schedule inside the scheduled work.*/ if (!sriov->is_going_down || id->scheduled_delete) { + /* + * Do not requeue timeout work that is already executing. The + * timeout handler may be waiting for id_map_lock and will free + * this entry after the caller releases it. + */ + if (id->scheduled_delete && + !cancel_delayed_work(&id->timeout)) + goto out; + id->scheduled_delete = 1; id->rtu_timeout = rtu_timeout; mod_delayed_work(cm_wq, &id->timeout, timeout); } +out: spin_unlock_irqrestore(&sriov->going_down_lock, flags); } From 9165b3f7372bc1547549687f3a113c047ed94a36 Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Wed, 14 Oct 2026 16:21:44 +0000 Subject: [PATCH 138/153] RDMA/irdma: Check MR contiguity before PBLE alloc The original logic would attempt to allocate PBLEs, then would check whether the region was contiguous _after_, and if so, would free the PBLEs and use a direct mapping instead. These unnecessary temporary PBLE allocations can technically fail (PBLEs are a finite resource), which can cause an MR registration failure even if the region is contiguous and should have succeeded. Fix this by checking whether the region is contiguous before allocating any PBLEs. This also provides an opportunity to consolidate some of the rereg_mr logic and use the existing ib_umem_is_contiguous() helper. Tested on GEN3 HW with a test program that does an extensive sweep of MR size/alignment/hugepage combinations, but most importantly, first/last byte reachable and -1/+1 unreachable when using contig regions with both aligned and unaligned start addresses, plus a kprobe on irdma_get_pble() to ensure we're actually hitting the new path. Ring MRs still have their own logic. That will be addressed next. Signed-off-by: Jacob Moroni Link: https://patch.msgid.link/20261014162144.4053279-1-jmoroni@google.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/verbs.c | 97 +++++++++-------------------- 1 file changed, 29 insertions(+), 68 deletions(-) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 71d7ae7baa47..f89c8516c57c 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -2917,42 +2917,6 @@ static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) return true; } -/** - * irdma_check_mr_contiguous - check if MR is physically contiguous - * @palloc: pbl allocation struct - * @pg_size: page size - */ -static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, - u32 pg_size) -{ - struct irdma_pble_level2 *lvl2 = &palloc->level2; - struct irdma_pble_info *leaf = lvl2->leaf; - u64 *arr = NULL; - u64 *start_addr = NULL; - int i; - bool ret; - - if (palloc->level == PBLE_LEVEL_1) { - arr = palloc->level1.addr; - ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, - pg_size); - return ret; - } - - start_addr = leaf->addr; - - for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { - arr = leaf->addr; - if ((*start_addr + ((u64)i * pg_size * PBLE_PER_PAGE)) != *arr) - return false; - ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); - if (!ret) - return false; - } - - return true; -} - /** * irdma_setup_pbles - copy user pg address to pble's * @rf: RDMA PCI function @@ -2989,9 +2953,6 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level); - if (lvl) - iwmr->pgaddrmem[0] = *pbl; - return 0; } @@ -3413,24 +3374,33 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, { struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct ib_umem *region = iwmr->region; u32 stag = 0; - u8 lvl; int err; - lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; iwmr->access = access; - err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); - if (err) - return err; - - if (lvl) { - err = irdma_check_mr_contiguous(&iwpbl->pble_alloc, + if (ib_umem_is_contiguous(region)) { + /* HW needs a valid page size even without PBLEs, so use 4K. It + * behaves like a PBLE-backed region where the PBA must be page + * aligned. HW takes the page offset from the lower 12 VA bits. + */ + iwmr->page_size = SZ_4K; + iwmr->pgaddrmem[0] = ALIGN_DOWN(ib_umem_start_dma_addr(region), iwmr->page_size); - if (err) { - irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); - iwpbl->pbl_allocated = false; - } + } else { + iwmr->page_size = ib_umem_find_best_pgsz(region, + iwdev->rf->sc_dev.hw_attrs.page_size_cap, + iwmr->ibmr.iova); + if (unlikely(!iwmr->page_size)) + return -EOPNOTSUPP; + + iwmr->page_cnt = ib_umem_num_dma_blocks(region, + iwmr->page_size); + err = irdma_setup_pbles(iwdev->rf, iwmr, + PBLE_LEVEL_1 | PBLE_LEVEL_2); + if (err) + return err; } if (create_stag) { @@ -3466,10 +3436,8 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, struct ib_pd *pd, u64 virt, enum irdma_memreg_type reg_type) { - struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_pbl *iwpbl; struct irdma_mr *iwmr; - unsigned long pgsz_bitmap; iwmr = kzalloc_obj(*iwmr); if (!iwmr) @@ -3483,18 +3451,21 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, iwmr->ibmr.device = pd->device; iwmr->ibmr.iova = virt; iwmr->type = reg_type; + iwmr->len = region->length; + iwpbl->user_base = virt; - pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ? - iwdev->rf->sc_dev.hw_attrs.page_size_cap : SZ_4K; + /* Page size and count are calculated in irdma_reg_user_mr_type_mem() + * for MEM type MRs. + */ + if (reg_type == IRDMA_MEMREG_TYPE_MEM) + return iwmr; - iwmr->page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt); + iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K, virt); if (unlikely(!iwmr->page_size)) { kfree(iwmr); return ERR_PTR(-EOPNOTSUPP); } - iwmr->len = region->length; - iwpbl->user_base = virt; iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); return iwmr; @@ -3833,7 +3804,6 @@ static int irdma_hwdereg_mr(struct ib_mr *ib_mr) static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, u64 virt) { - struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct ib_pd *pd = iwmr->ibmr.pd; struct ib_umem *region; @@ -3846,17 +3816,8 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, iwmr->region = region; iwmr->ibmr.iova = virt; iwmr->ibmr.pd = pd; - iwmr->page_size = ib_umem_find_best_pgsz(region, - iwdev->rf->sc_dev.hw_attrs.page_size_cap, - virt); - if (unlikely(!iwmr->page_size)) { - err = -EOPNOTSUPP; - goto err; - } - iwmr->len = region->length; iwpbl->user_base = virt; - iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false); if (err) From dd36137b218ad6f18ed15e6279412d7fc160980c Mon Sep 17 00:00:00 2001 From: Serhat Kumral Date: Thu, 15 Oct 2026 13:25:18 +0300 Subject: [PATCH 139/153] IB/rdmavt: Fix error unwind path for qp_priv_init() in rvt_create_qp() If driver_f.qp_priv_init() fails, execution jumps to bail_rq_wq, which bypasses bail_qpn. Because alloc_qpn() has already succeeded at this point, the allocated QP number is left unreleased. Jump to bail_qpn instead to ensure the QPN is properly freed via rvt_free_qpn(). Fixes: 5190f052a3654 ("IB/hfi1: Allow the driver to initialize QP priv struct") Signed-off-by: Serhat Kumral Link: https://patch.msgid.link/20261015102518.15780-1-serhatkumral1@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rdmavt/qp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index d8ee485c49a6..3602b9ff2196 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -1173,7 +1173,7 @@ int rvt_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, if (rdi->driver_f.qp_priv_init) { ret = rdi->driver_f.qp_priv_init(rdi, qp, init_attr); if (ret) - goto bail_rq_wq; + goto bail_qpn; } break; From 59df57262e6fa9e6f4aa76d6d324134777c702a0 Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Thu, 15 Oct 2026 16:17:55 +0000 Subject: [PATCH 140/153] RDMA/irdma: Fix FRWR IRDMAQPSQ_PBLADDR field population Previously, the IRDMAQPSQ_PBLADDR FRWR WQE field was being populated using a mask that was right-shifted, whereas it should have been the address being right-shifted. This ended up working fine in many cases because the lowermost 12 bits are always zero (the addresses are page aligned) and the uppermost 12 bits are usually zero too, but it could break for DMA addresses that are wider than 52 bits. Fix this by right-shifting the address instead of the mask, similar to how it was done in the original i40iw driver. Tested with NFS over RDMA with some added debug logs to ensure it was hitting both the direct (single page) path as well as multi page (< 512). Fixes: 3f49d6842569 ("RDMA/irdma: Implement HW Admin Queue OPs") Signed-off-by: Jacob Moroni Link: https://patch.msgid.link/20261015161756.3078300-1-jmoroni@google.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/ctrl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c index 0db73fd7d01b..b62bf4d11829 100644 --- a/drivers/infiniband/hw/irdma/ctrl.c +++ b/drivers/infiniband/hw/irdma/ctrl.c @@ -1727,7 +1727,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, info->first_pm_pbl_index >> 16); set_64bit_val(wqe, 8, FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) | - FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa)); + FIELD_PREP(IRDMAQPSQ_PBLADDR, info->reg_addr_pa >> IRDMA_HW_PAGE_SHIFT)); set_64bit_val(wqe, 16, info->total_len | FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index)); From 4a953ee228f5544a3e5515eed88bf09d1a5adb94 Mon Sep 17 00:00:00 2001 From: Deep Shah Date: Thu, 15 Oct 2026 08:36:36 +0000 Subject: [PATCH 141/153] RDMA/uverbs: Fix uobject leak in DMABUF release uverbs_dmabuf_release() treats a NULL uobj->context as an aborted allocation, but normal ufile cleanup also clears it. Closing the uverbs FD before the dma-buf FD therefore skips uverbs_uobject_release() and leaks the uobject and ufile references. Clear dmabuf->priv before dma_buf_put() on allocation failure and use that NULL pointer to identify an abort. Otherwise, always call uverbs_uobject_release(). This also avoids reading a freed uobject if release is deferred until after the allocation has been aborted. Fixes: 0ac6f4056c4a ("RDMA/uverbs: Add DMABUF object type and operations") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Deep Shah Link: https://patch.msgid.link/20261015083637.200958-2-deepshah146@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/uverbs_std_types_dmabuf.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/uverbs_std_types_dmabuf.c b/drivers/infiniband/core/uverbs_std_types_dmabuf.c index 2411ebee69e2..d88d30f1e42f 100644 --- a/drivers/infiniband/core/uverbs_std_types_dmabuf.c +++ b/drivers/infiniband/core/uverbs_std_types_dmabuf.c @@ -68,9 +68,10 @@ static void uverbs_dmabuf_release(struct dma_buf *dmabuf) struct ib_uverbs_dmabuf_file *priv = dmabuf->priv; /* - * This can only happen if the fput came from alloc_abort_fd_uobject() + * This can only happen if the fput came from the error unwind of + * UVERBS_METHOD_DMABUF_ALLOC. The abort cleans up the uobject. */ - if (!priv->uobj.context) + if (!priv) return; uverbs_uobject_release(&priv->uobj); @@ -141,6 +142,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DMABUF_ALLOC)( return 0; err_revoked: + /* Tell uverbs_dmabuf_release() that the uobject is being aborted */ + uverbs_dmabuf->dmabuf->priv = NULL; dma_buf_put(uverbs_dmabuf->dmabuf); err: rdma_user_mmap_entry_put(mmap_entry); From 2854f9025d4ef6c4566f71e52d7f496d9a0f5737 Mon Sep 17 00:00:00 2001 From: Deep Shah Date: Thu, 15 Oct 2026 08:36:37 +0000 Subject: [PATCH 142/153] RDMA/uverbs: Fix DMABUF use-after-free during ufile cleanup The DMABUF release can race with ufile cleanup. When the cleanup holds hw_destroy_rwsem for write, uverbs_uobject_release() fails its trylock and leaves the uobject on the ufile list, but the dma_buf is freed as soon as the release returns. The cleanup then destroys the uobject and uses the freed dma_buf: BUG: KASAN: slab-use-after-free in uverbs_dmabuf_fd_destroy_uobj+0x46/0x200 Read of size 8 at addr ff11000109cf14b0 by task repro/435 Call Trace: uverbs_dmabuf_fd_destroy_uobj+0x46/0x200 destroy_hw_fd_uobject+0x3f/0x50 uverbs_destroy_uobject+0xc5/0x380 __uverbs_cleanup_ufile+0x1a8/0x260 uverbs_destroy_ufile_hw+0x54/0x160 ib_uverbs_close+0x34/0x110 Allocated by task 435: dma_buf_export+0x1b5/0x530 ib_uverbs_handler_UVERBS_METHOD_DMABUF_ALLOC+0x223/0x3f0 Freed by task 436: kfree+0x301/0x530 dentry_kill+0x14c/0x560 __fput+0x23a/0x4a0 rdma_user_mmap_entry_remove() dereferences the same pointer. Protect the stored dma_buf pointer with a mutex and clear it before release returns. Use a shared revoke helper so uobject destruction and mmap entry removal both skip a cleared pointer. Hold a temporary uobject reference across uverbs_uobject_release() and the pointer update. Use a separate mutex because uobject destruction can free the mmap entry before release finishes. Map and unmap use attachment->dmabuf directly, so they do not need to take the new mutex while holding the reservation lock. Fixes: 0ac6f4056c4a ("RDMA/uverbs: Add DMABUF object type and operations") Cc: stable@vger.kernel.org Assisted-by: LLM Signed-off-by: Deep Shah Link: https://patch.msgid.link/20261015083637.200958-3-deepshah146@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/ib_core_uverbs.c | 10 +---- drivers/infiniband/core/uverbs.h | 37 ++++++++++++++++++ .../infiniband/core/uverbs_std_types_dmabuf.c | 39 +++++++++++-------- 3 files changed, 61 insertions(+), 25 deletions(-) diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index 1482d9b27b38..74d9c006bad5 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -244,16 +244,8 @@ void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry) entry->driver_removed = true; xa_unlock(&entry->ucontext->mmap_xa); list_for_each_entry_safe(uverbs_dmabuf, tmp, &entry->dmabufs, dmabufs_elm) { - dma_resv_lock(uverbs_dmabuf->dmabuf->resv, NULL); list_del(&uverbs_dmabuf->dmabufs_elm); - uverbs_dmabuf->revoked = true; - dma_buf_invalidate_mappings(uverbs_dmabuf->dmabuf); - dma_resv_wait_timeout(uverbs_dmabuf->dmabuf->resv, - DMA_RESV_USAGE_BOOKKEEP, false, - MAX_SCHEDULE_TIMEOUT); - dma_resv_unlock(uverbs_dmabuf->dmabuf->resv); - kref_put(&uverbs_dmabuf->kref, ib_uverbs_dmabuf_done); - wait_for_completion(&uverbs_dmabuf->comp); + ib_uverbs_dmabuf_revoke(uverbs_dmabuf); } mutex_unlock(&entry->dmabufs_lock); diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index c64dd6b94e10..728e09b41992 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -42,6 +42,8 @@ #include #include #include +#include +#include #include #include @@ -135,6 +137,13 @@ struct ib_uverbs_completion_event_file { struct ib_uverbs_dmabuf_file { struct ib_uobject uobj; + /* + * Serializes users of dmabuf with the dma_buf's release, which sets it + * to NULL. The uobject can outlive the dma_buf while it waits to be + * destroyed. The map and unmap callbacks use attachment->dmabuf + * instead of taking this lock. + */ + struct mutex dmabuf_release_lock; struct dma_buf *dmabuf; struct list_head dmabufs_elm; struct rdma_user_mmap_entry *mmap_entry; @@ -359,6 +368,34 @@ static inline void ib_uverbs_dmabuf_done(struct kref *kref) complete(&priv->comp); } +/* + * Revoke the dmabuf and wait for all of its DMA mappings to be gone. Must be + * called with the mmap entry's dmabufs_lock held, after the dmabuf has been + * removed from the entry's dmabufs list, so it runs once per dmabuf. Lock + * order is dmabufs_lock, dmabuf_release_lock, then the dma_resv lock. + */ +static inline void ib_uverbs_dmabuf_revoke(struct ib_uverbs_dmabuf_file *priv) +{ + mutex_lock(&priv->dmabuf_release_lock); + if (priv->dmabuf) { + dma_resv_lock(priv->dmabuf->resv, NULL); + priv->revoked = true; + dma_buf_invalidate_mappings(priv->dmabuf); + dma_resv_wait_timeout(priv->dmabuf->resv, + DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_unlock(priv->dmabuf->resv); + } else { + /* A released dma_buf has no attachments left to invalidate */ + priv->revoked = true; + } + mutex_unlock(&priv->dmabuf_release_lock); + + kref_put(&priv->kref, ib_uverbs_dmabuf_done); + /* Wait till all DMA unmaps are completed */ + wait_for_completion(&priv->comp); +} + int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, enum rdma_remove_reason reason); diff --git a/drivers/infiniband/core/uverbs_std_types_dmabuf.c b/drivers/infiniband/core/uverbs_std_types_dmabuf.c index d88d30f1e42f..8d8c997a4883 100644 --- a/drivers/infiniband/core/uverbs_std_types_dmabuf.c +++ b/drivers/infiniband/core/uverbs_std_types_dmabuf.c @@ -28,7 +28,7 @@ uverbs_dmabuf_map(struct dma_buf_attachment *attachment, struct ib_uverbs_dmabuf_file *priv = attachment->dmabuf->priv; struct sg_table *ret; - dma_resv_assert_held(priv->dmabuf->resv); + dma_resv_assert_held(attachment->dmabuf->resv); if (priv->revoked) return ERR_PTR(-ENODEV); @@ -49,7 +49,7 @@ static void uverbs_dmabuf_unmap(struct dma_buf_attachment *attachment, { struct ib_uverbs_dmabuf_file *priv = attachment->dmabuf->priv; - dma_resv_assert_held(priv->dmabuf->resv); + dma_resv_assert_held(attachment->dmabuf->resv); dma_buf_free_sgt(attachment, sgt, dir); kref_put(&priv->kref, ib_uverbs_dmabuf_done); } @@ -74,7 +74,25 @@ static void uverbs_dmabuf_release(struct dma_buf *dmabuf) if (!priv) return; + /* + * uverbs_uobject_release() may drop the last reference to the uobject, + * keep it around until the dmabuf pointer below has been cleared. + */ + uverbs_uobject_get(&priv->uobj); uverbs_uobject_release(&priv->uobj); + + /* + * The dma_buf is freed once this function returns. If the uobject was + * not destroyed above, because the ufile is concurrently being cleaned + * up, it is still reachable through the ufile's uobjects list and the + * mmap entry's dmabufs list. Make sure those paths are not in the + * middle of using the dma_buf and cannot use it afterwards. + */ + mutex_lock(&priv->dmabuf_release_lock); + priv->dmabuf = NULL; + mutex_unlock(&priv->dmabuf_release_lock); + + uverbs_uobject_put(&priv->uobj); } static const struct dma_buf_ops uverbs_dmabuf_ops = { @@ -100,6 +118,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DMABUF_ALLOC)( off_t pg_off; int ret; + mutex_init(&uverbs_dmabuf->dmabuf_release_lock); + ret = uverbs_get_const(&pg_off, attrs, UVERBS_ATTR_ALLOC_DMABUF_PGOFF); if (ret) return ret; @@ -165,24 +185,11 @@ static void uverbs_dmabuf_fd_destroy_uobj(struct ib_uobject *uobj, { struct ib_uverbs_dmabuf_file *uverbs_dmabuf = container_of(uobj, struct ib_uverbs_dmabuf_file, uobj); - bool wait_for_comp = false; mutex_lock(&uverbs_dmabuf->mmap_entry->dmabufs_lock); - dma_resv_lock(uverbs_dmabuf->dmabuf->resv, NULL); if (!uverbs_dmabuf->revoked) { - uverbs_dmabuf->revoked = true; list_del(&uverbs_dmabuf->dmabufs_elm); - dma_buf_invalidate_mappings(uverbs_dmabuf->dmabuf); - dma_resv_wait_timeout(uverbs_dmabuf->dmabuf->resv, - DMA_RESV_USAGE_BOOKKEEP, false, - MAX_SCHEDULE_TIMEOUT); - wait_for_comp = true; - } - dma_resv_unlock(uverbs_dmabuf->dmabuf->resv); - if (wait_for_comp) { - kref_put(&uverbs_dmabuf->kref, ib_uverbs_dmabuf_done); - /* Let's wait till all DMA unmap are completed. */ - wait_for_completion(&uverbs_dmabuf->comp); + ib_uverbs_dmabuf_revoke(uverbs_dmabuf); } mutex_unlock(&uverbs_dmabuf->mmap_entry->dmabufs_lock); From d01ee4c84193196b6e4be4caf1ff8fdcc441ed96 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 18 Oct 2026 07:59:38 -0400 Subject: [PATCH 143/153] RDMA/core: Preserve the selected source GID on failed AH modification When an address-handle modification fails, the core replaces its source-GID reference. The old address then loses the reference which keeps its source entry alive. Fixes: 1a1f460ff151 ("RDMA: Hold the sgid_attr inside the struct ib_ah/qp") Link: https://patch.msgid.link/20261015-ib-core-fixes-v1-1-60f6da3ddec3@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/verbs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index edf62652f9d3..7d52d3036208 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1011,7 +1011,8 @@ int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr) ah->device->ops.modify_ah(ah, ah_attr) : -EOPNOTSUPP; - ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr); + if (!ret) + ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr); rdma_unfill_sgid_attr(ah_attr, old_sgid_attr); return ret; } From d74bdcfa89fa45d3a506cddc37b3bf1b76cea0ce Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 18 Oct 2026 07:59:38 -0400 Subject: [PATCH 144/153] RDMA/core: Free unpublished port statistics after initial-read failure An initial port counter read can fail after allocating its statistics buffer and wrapper. Free these both allocations. Fixes: 467f432a521a ("RDMA/core: Split port and device counter sysfs attributes") Link: https://patch.msgid.link/20261015-ib-core-fixes-v1-2-60f6da3ddec3@nvidia.com Reviewed-by: Kalesh AP Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/sysfs.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index 69df8b703cbd..4cd861b5d953 100644 --- a/drivers/infiniband/core/sysfs.c +++ b/drivers/infiniband/core/sysfs.c @@ -996,8 +996,8 @@ static int setup_hw_port_stats(struct ib_port *port, data->stats->num_counters); if (ret != data->stats->num_counters) { if (WARN_ON(ret >= 0)) - return -EINVAL; - return ret; + ret = -EINVAL; + goto err_free_data; } data->stats->timestamp = jiffies; @@ -1031,6 +1031,11 @@ static int setup_hw_port_stats(struct ib_port *port, port->hw_stats_data = data; return 0; + +err_free_data: + rdma_free_hw_stats_struct(data->stats); + kfree(data); + return ret; } struct rdma_hw_stats *ib_get_hw_stats_port(struct ib_device *ibdev, From fdfd099222d9de9e2f1b8d69e94494c5489272cc Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 18 Oct 2026 07:59:38 -0400 Subject: [PATCH 145/153] RDMA/core: Reject LAG selection when source MAC lookup fails GID deletion can clear the source device association after LAG lookup retains the bonding master. Ignoring the resulting L2 lookup failure copies uninitialized source MAC bytes into the synthetic hash packet. Read the source MAC before allocating the packet and propagate lookup errors to the caller, so failed lookups cannot influence slave selection. Fixes: bd3920eac133 ("RDMA/core: Add LAG functionality") Link: https://patch.msgid.link/20261015-ib-core-fixes-v1-3-60f6da3ddec3@nvidia.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/lag.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c index 88aac911e8ee..651c868aa04e 100644 --- a/drivers/infiniband/core/lag.c +++ b/drivers/infiniband/core/lag.c @@ -19,6 +19,11 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev, u8 smac[ETH_ALEN]; bool is_ipv4; int hdr_len; + int ret; + + ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, NULL, smac); + if (ret) + return ERR_PTR(ret); is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw); hdr_len = ETH_HLEN + sizeof(struct udphdr) + LL_RESERVED_SPACE(netdev); @@ -26,7 +31,7 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev, skb = alloc_skb(hdr_len, flags); if (!skb) - return NULL; + return ERR_PTR(-ENOMEM); skb->dev = netdev; skb_reserve(skb, hdr_len); @@ -70,7 +75,6 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev, skb_reset_mac_header(skb); eth = eth_hdr(skb); skb->protocol = eth->h_proto = htons(is_ipv4 ? ETH_P_IP : ETH_P_IPV6); - rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, NULL, smac); memcpy(eth->h_source, smac, ETH_ALEN); memcpy(eth->h_dest, ah_attr->roce.dmac, ETH_ALEN); @@ -86,8 +90,8 @@ static struct net_device *rdma_get_xmit_slave_udp(struct ib_device *device, struct sk_buff *skb; skb = rdma_build_skb(master, ah_attr, flags); - if (!skb) - return ERR_PTR(-ENOMEM); + if (IS_ERR(skb)) + return ERR_CAST(skb); rcu_read_lock(); slave = netdev_get_xmit_slave(master, skb, From 9ffd2aa8b53b044a61f939e8fba5ec250c5fddd9 Mon Sep 17 00:00:00 2001 From: Jacob Moroni Date: Fri, 16 Oct 2026 18:09:01 +0000 Subject: [PATCH 146/153] RDMA/irdma: Fix PBLE_LEVEL_2 FRWR generation When an FRWR is posted with only 1 page mapped, it uses a direct mapping, with the page-aligned DMA address provided in the WQE (reg_addr_pa). This address is obtained by reading it from the PBLE that was previously populated during ib_map_mr_sg(), but it did not account for whether the original allocation was PBLE_LEVEL_1 or PBLE_LEVEL_2 and just assumed it was always level 1. As a result, it would program incorrect addresses if the MR was originally allocated as PBLE_LEVEL_2. Additionally, multi-page PBLE_LEVEL_2 FRWRs were also broken because it was using the wrong chunk_size. Fix these issues by correctly populating the stag_info fields depending on whether it's a PBLE_LEVEL_1 or PBLE_LEVEL_2 MR and whether more than 1 page has been mapped or not. Tested with NFS over RDMA running various fio scenarios, with and without a modified kernel to force it to take the PBLE_LEVEL_2 path. Fixes: 137d264c6f63 ("RDMA/irdma: Add 2 level PBLE support for FMR") Suggested-by: Stefan Metzmacher Link: https://lore.kernel.org/linux-rdma/20261012161135.494544-1-metze@samba.org/ Signed-off-by: Jacob Moroni Link: https://patch.msgid.link/20261016180901.3292775-1-jmoroni@google.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/verbs.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index f89c8516c57c..c91c55917de9 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -4247,11 +4247,23 @@ static int irdma_post_send(struct ib_qp *ibqp, stag_info.wr_id = ib_wr->wr_id; stag_info.va = iwmr->ibmr.iova; stag_info.total_len = iwmr->ibmr.length; - stag_info.reg_addr_pa = *palloc->level1.addr; - stag_info.first_pm_pbl_index = palloc->level1.idx; stag_info.local_fence = true; - if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) - stag_info.chunk_size = 1; + if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) { + if (palloc->level == PBLE_LEVEL_1) { + stag_info.first_pm_pbl_index = + palloc->level1.idx; + stag_info.chunk_size = 1; + } else { + stag_info.first_pm_pbl_index = + palloc->level2.root.idx; + stag_info.chunk_size = 3; + } + } else { + stag_info.reg_addr_pa = + (palloc->level == PBLE_LEVEL_1) ? + *palloc->level1.addr : + *palloc->level2.leaf->addr; + } err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info); break; From 4b8f20dc8d25f575e8b8bb5abb09ab803a7e1a3f Mon Sep 17 00:00:00 2001 From: Serhat Kumral Date: Sat, 17 Oct 2026 00:15:09 +0300 Subject: [PATCH 147/153] RDMA/srp: Avoid reading uninitialized has_port on srp_parse_in() failure srp_parse_in() returns -ENOMEM without initializing *has_port if kstrdup() fails. Only check has_port when srp_parse_in() succeeds. Fixes: bcef5b721568 ("RDMA/srp: Accept again source addresses that do not have a port number") Signed-off-by: Serhat Kumral Link: https://patch.msgid.link/20261016211509.12973-1-serhatkumral1@gmail.com Reviewed-by: Bart Van Assche Signed-off-by: Leon Romanovsky --- drivers/infiniband/ulp/srp/ib_srp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 955f36efeebd..5811f02df980 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -3499,7 +3499,7 @@ static int srp_parse_options(struct net *net, const char *buf, } ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p, &has_port); - if (!has_port) + if (ret == 0 && !has_port) ret = -EINVAL; if (ret < 0) { pr_warn("bad dest parameter '%s'\n", p); From d89ac72bd2deff15756cde6d5d70b41ae1b94290 Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Fri, 16 Oct 2026 17:39:34 -0500 Subject: [PATCH 148/153] dt-bindings: infiniband: Convert hisilicon,hns-roce-v1 to DT schema Convert hisilicon,hns-roce-v1 binding to DT schema format. It's a straight forward conversion. While generally '#address-cells' and '#size-cells' are not needed in leaf nodes, there are some exceptions. Not sure if there's a dependency in this case, so leaving them in place since this is an old binding. Signed-off-by: Rob Herring (Arm) Link: https://patch.msgid.link/20261016223935.1496846-1-robh@kernel.org Signed-off-by: Leon Romanovsky --- .../infiniband/hisilicon,hns-roce-v1.yaml | 173 ++++++++++++++++++ .../infiniband/hisilicon-hns-roce.txt | 108 ----------- MAINTAINERS | 2 +- 3 files changed, 174 insertions(+), 109 deletions(-) create mode 100644 Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml delete mode 100644 Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt diff --git a/Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml b/Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml new file mode 100644 index 000000000000..d987840f417c --- /dev/null +++ b/Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml @@ -0,0 +1,173 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/infiniband/hisilicon,hns-roce-v1.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Hisilicon HNS RoCE + +maintainers: + - Chengchang Tang + - Junxian Huang + +description: + Hisilicon RoCE engine is a part of network subsystem. It works depending on + other part of network subsystem, such as gmac and dsa fabric. + +properties: + compatible: + const: hisilicon,hns-roce-v1 + + reg: + maxItems: 1 + + '#address-cells': + const: 2 + + '#size-cells': + const: 2 + + dma-coherent: true + + interrupts: + items: + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Async event IRQ + - description: Event overflow IRQ + + interrupt-names: + items: + - const: hns-roce-comp-0 + - const: hns-roce-comp-1 + - const: hns-roce-comp-2 + - const: hns-roce-comp-3 + - const: hns-roce-comp-4 + - const: hns-roce-comp-5 + - const: hns-roce-comp-6 + - const: hns-roce-comp-7 + - const: hns-roce-comp-8 + - const: hns-roce-comp-9 + - const: hns-roce-comp-10 + - const: hns-roce-comp-11 + - const: hns-roce-comp-12 + - const: hns-roce-comp-13 + - const: hns-roce-comp-14 + - const: hns-roce-comp-15 + - const: hns-roce-comp-16 + - const: hns-roce-comp-17 + - const: hns-roce-comp-18 + - const: hns-roce-comp-19 + - const: hns-roce-comp-20 + - const: hns-roce-comp-21 + - const: hns-roce-comp-22 + - const: hns-roce-comp-23 + - const: hns-roce-comp-24 + - const: hns-roce-comp-25 + - const: hns-roce-comp-26 + - const: hns-roce-comp-27 + - const: hns-roce-comp-28 + - const: hns-roce-comp-29 + - const: hns-roce-comp-30 + - const: hns-roce-comp-31 + - const: hns-roce-async + - const: hns-roce-common + + eth-handle: + description: References to one or more Ethernet devices. + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + maxItems: 1 + + dsaf-handle: + description: Reference to a dsaf device. + $ref: /schemas/types.yaml#/definitions/phandle + + node-guid: + description: A number that uniquely identifies a device or component. + $ref: /schemas/types.yaml#/definitions/uint8-array + +required: + - compatible + - reg + - '#address-cells' + - '#size-cells' + - eth-handle + - dsaf-handle + - node-guid + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + infiniband@c4000000 { + compatible = "hisilicon,hns-roce-v1"; + reg = <0x0 0xc4000000 0x0 0x100000>; + #address-cells = <2>; + #size-cells = <2>; + dma-coherent; + eth-handle = <ð2 ð3 ð4 ð5 ð6 ð7>; + dsaf-handle = <&soc0_dsa>; + node-guid = [00 9A CD 00 00 01 02 03]; + interrupts = <722 1>, <723 1>, <724 1>, <725 1>, + <726 1>, <727 1>, <728 1>, <729 1>, + <730 1>, <731 1>, <732 1>, <733 1>, + <734 1>, <735 1>, <736 1>, <737 1>, + <738 1>, <739 1>, <740 1>, <741 1>, + <742 1>, <743 1>, <744 1>, <745 1>, + <746 1>, <747 1>, <748 1>, <749 1>, + <750 1>, <751 1>, <752 1>, <753 1>, + <785 1>, <754 4>; + interrupt-names = "hns-roce-comp-0", "hns-roce-comp-1", + "hns-roce-comp-2", "hns-roce-comp-3", + "hns-roce-comp-4", "hns-roce-comp-5", + "hns-roce-comp-6", "hns-roce-comp-7", + "hns-roce-comp-8", "hns-roce-comp-9", + "hns-roce-comp-10", "hns-roce-comp-11", + "hns-roce-comp-12", "hns-roce-comp-13", + "hns-roce-comp-14", "hns-roce-comp-15", + "hns-roce-comp-16", "hns-roce-comp-17", + "hns-roce-comp-18", "hns-roce-comp-19", + "hns-roce-comp-20", "hns-roce-comp-21", + "hns-roce-comp-22", "hns-roce-comp-23", + "hns-roce-comp-24", "hns-roce-comp-25", + "hns-roce-comp-26", "hns-roce-comp-27", + "hns-roce-comp-28", "hns-roce-comp-29", + "hns-roce-comp-30", "hns-roce-comp-31", + "hns-roce-async", "hns-roce-common"; + }; + }; diff --git a/Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt b/Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt deleted file mode 100644 index be31cf05cd2e..000000000000 --- a/Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt +++ /dev/null @@ -1,108 +0,0 @@ -Hisilicon RoCE DT description - -Hisilicon RoCE engine is a part of network subsystem. -It works depending on other part of network subsystem, such as gmac and -dsa fabric. - -Additional properties are described here: - -Required properties: -- compatible: Should contain "hisilicon,hns-roce-v1". -- reg: Physical base address of the RoCE driver and -length of memory mapped region. -- eth-handle: phandle, specifies a reference to a node -representing a ethernet device. -- dsaf-handle: phandle, specifies a reference to a node -representing a dsaf device. -- node_guid: a number that uniquely identifies a device or component -- #address-cells: must be 2 -- #size-cells: must be 2 -Optional properties: -- dma-coherent: Present if DMA operations are coherent. -- interrupts: should contain 32 completion event irq,1 async event irq -and 1 event overflow irq. -- interrupt-names:should be one of 34 irqs for roce device - - hns-roce-comp-0 ~ hns-roce-comp-31: 32 complete event irq - - hns-roce-async: 1 async event irq - - hns-roce-common: named common exception warning irq -Example: - infiniband@c4000000 { - compatible = "hisilicon,hns-roce-v1"; - reg = <0x0 0xc4000000 0x0 0x100000>; - dma-coherent; - eth-handle = <ð2 ð3 ð4 ð5 ð6 ð7>; - dsaf-handle = <&soc0_dsa>; - node-guid = [00 9A CD 00 00 01 02 03]; - #address-cells = <2>; - #size-cells = <2>; - interrupt-parent = <&mbigen_dsa>; - interrupts = <722 1>, - <723 1>, - <724 1>, - <725 1>, - <726 1>, - <727 1>, - <728 1>, - <729 1>, - <730 1>, - <731 1>, - <732 1>, - <733 1>, - <734 1>, - <735 1>, - <736 1>, - <737 1>, - <738 1>, - <739 1>, - <740 1>, - <741 1>, - <742 1>, - <743 1>, - <744 1>, - <745 1>, - <746 1>, - <747 1>, - <748 1>, - <749 1>, - <750 1>, - <751 1>, - <752 1>, - <753 1>, - <785 1>, - <754 4>; - - interrupt-names = "hns-roce-comp-0", - "hns-roce-comp-1", - "hns-roce-comp-2", - "hns-roce-comp-3", - "hns-roce-comp-4", - "hns-roce-comp-5", - "hns-roce-comp-6", - "hns-roce-comp-7", - "hns-roce-comp-8", - "hns-roce-comp-9", - "hns-roce-comp-10", - "hns-roce-comp-11", - "hns-roce-comp-12", - "hns-roce-comp-13", - "hns-roce-comp-14", - "hns-roce-comp-15", - "hns-roce-comp-16", - "hns-roce-comp-17", - "hns-roce-comp-18", - "hns-roce-comp-19", - "hns-roce-comp-20", - "hns-roce-comp-21", - "hns-roce-comp-22", - "hns-roce-comp-23", - "hns-roce-comp-24", - "hns-roce-comp-25", - "hns-roce-comp-26", - "hns-roce-comp-27", - "hns-roce-comp-28", - "hns-roce-comp-29", - "hns-roce-comp-30", - "hns-roce-comp-31", - "hns-roce-async", - "hns-roce-common"; - }; diff --git a/MAINTAINERS b/MAINTAINERS index 31699b112d3e..2a6e6ec629e0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11867,7 +11867,7 @@ M: Chengchang Tang M: Junxian Huang L: linux-rdma@vger.kernel.org S: Maintained -F: Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt +F: Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml F: drivers/infiniband/hw/hns/ HISILICON SAS Controller From 7d7214b39b14c9763c14445b139f63369063b65e Mon Sep 17 00:00:00 2001 From: Zhiwei Zhang <202275009@qq.com> Date: Sat, 17 Oct 2026 09:40:21 +0800 Subject: [PATCH 149/153] RDMA/rxe: Clear pkt->qp before the early drop exits in rxe_rcv() rxe_rcv() puts pkt->qp at its drop label, but leaves pkt->qp uninitialized on the two paths that take an early goto: if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop; if (rxe_chk_dgid(rxe, skb) < 0) goto drop; pkt->opcode = bth_opcode(pkt); pkt->psn = bth_psn(pkt); pkt->qp = NULL; <- too late for the gotos above ... drop: if (pkt->qp) rxe_put(pkt->qp); pkt points into skb->cb, which rxe does not own at this point: the network stack has used that space for its own state. For IPv6, ip6_rcv_core() memsets inet6_skb_parm and then writes IP6CB(skb)->nhoff = offsetof(struct ipv6hdr, nexthdr); at offset 14, so the qword at cb + 8, which is where pkt->qp lives, reads back as 0x0006000000000000 for every IPv6 packet. Any IPv6 datagram to port 4791 whose payload is shorter than the BTH takes the first goto and reaches the drop label with that value, so rxe_put() runs refcount_dec_and_test() on 0x0006000000000150 and the kernel faults, in softirq, on a non-canonical address: general protection fault, probably for non-canonical address 0x0006000000000150: 0000 [#1] PREEMPT SMP RIP: 0010:__rxe_put+0x.../... Call Trace: rxe_rcv+0x.../... [rdma_rxe] rxe_udp_encap_recv+0x.../... [rdma_rxe] udp_queue_rcv_one_skb ... Kernel panic - not syncing: Fatal exception in interrupt The fault happens in softirq, so it panics even with panic_on_oops=0, and with panic=0 the machine hangs until it is power cycled. A remote sender only needs to send an empty UDP datagram to port 4791; no RDMA connection, GID or other state is involved and the rxe socket is bound in init_net. IPv4 is not affected: ip_rcv_core() memsets the whole inet_skb_parm, so the pkt->qp slot is zero there. Fix by clearing pkt->qp before the first early exit. pkt->qp is only ever assigned a reference-owning value by hdr_check() and rxe_rcv_mcast_pkt(), both of which run after this point, so the put at the drop label stays balanced. The ordering has been unsafe since the driver was merged: the two early gotos have preceded pkt->qp = NULL in every version since 8700e3e7c485. Testing done: the driver was built and run on a 6.2 vendor tree that carries this code, where a single 0-byte IPv6 datagram to port 4791 reproduced the panic above (that tree does not pull the UDP header in rxe_udp_encap_recv(), so payloads of 0..3 bytes reach the first goto there, versus 0..11 on mainline). With this change, the same packets are dropped as intended and tracing the drop label shows pkt->qp is 0 on every path that reaches it. Not done: the mainline tree was not built in the environment used to prepare this, and the reproducer is left out of this message on purpose - both are available on request. Fixes: 8700e3e7c485 ("Soft RoCE driver") Reviewed-by: Zhu Yanjun Signed-off-by: Zhiwei Zhang <202275009@qq.com> Link: https://patch.msgid.link/tencent_FBB0EE9F592FD21FD4B0F3BFAD3336765209@qq.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_recv.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index 2c90387aa126..e1d90633f085 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -320,6 +320,14 @@ void rxe_rcv(struct sk_buff *skb) struct rxe_dev *rxe = pkt->rxe; unsigned int skblen = skb->len - skb_network_offset(skb); + /* + * pkt lives in skb->cb, which the network stack has already used + * for its own state: for IPv6 the qword holding pkt->qp holds + * inet6_skb_parm.nhoff. The drop label puts pkt->qp, so clear + * it before the first early exit. + */ + pkt->qp = NULL; + if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop; @@ -328,7 +336,6 @@ void rxe_rcv(struct sk_buff *skb) pkt->opcode = bth_opcode(pkt); pkt->psn = bth_psn(pkt); - pkt->qp = NULL; pkt->mask |= rxe_opcode[pkt->opcode].mask; /* From 9dd8b9a337af844291edcd9139c927d0f7958dbe Mon Sep 17 00:00:00 2001 From: Serhat Kumral Date: Sat, 17 Oct 2026 21:50:28 +0300 Subject: [PATCH 150/153] RDMA/usnic: Bound buffer size to sizeof(buf) in flowinfo_read() In flowinfo_read(), 'left' tracks the remaining space in the 512-byte stack buffer 'buf', but it is initialized to 'count'. Initialize 'left' to sizeof(buf) instead of 'count'. Fixes: 9f637f793602 ("IB/usnic: Expose flows via debugfs") Signed-off-by: Serhat Kumral Link: https://patch.msgid.link/20261017185028.1019445-1-serhatkumral1@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/usnic/usnic_debugfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/usnic/usnic_debugfs.c b/drivers/infiniband/hw/usnic/usnic_debugfs.c index 10a8cd5ba076..ece7c6e71499 100644 --- a/drivers/infiniband/hw/usnic/usnic_debugfs.c +++ b/drivers/infiniband/hw/usnic/usnic_debugfs.c @@ -76,7 +76,7 @@ static ssize_t flowinfo_read(struct file *f, char __user *data, qp_flow = f->private_data; ptr = buf; - left = count; + left = sizeof(buf); if (*ppos > 0) return 0; From 0fe31052d948a3a993bb2f58cd5886f7ec279ea7 Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Mon, 19 Oct 2026 13:05:03 +0800 Subject: [PATCH 151/153] RDMA/mlx5: Fix CQ resize destination index mask In copy_resize_cqes, the destination CQE index is computed as (i + 1) & cq->resize_buf->nent, but resize_buf->nent is produced by roundup_pow_of_two(), so it is a power of two with exactly one bit set. Masking with a single-bit value can only yield 0 or nent itself. The buffer has nent entries, so valid indices are 0 .. nent - 1 and nent is one past the last slot; whenever the masked result is nent the following memcpy writes one entry out of bounds. Use (nent - 1) as the wrap mask instead. For a power-of-two nent, (i + 1) & (nent - 1) is the correct modulo-nent reduction, keeping the index within [0, nent - 1]. This matches get_sw_cqe(), which addresses the CQ buffer via n & cq->ibcq.cqe, where cq->ibcq.cqe == entries - 1 is exactly the nent - 1 wrap mask. Fixes: bde51583f49b ("IB/mlx5: Add support for resize CQ") Signed-off-by: Li RongQing Link: https://patch.msgid.link/20261019050503.2313-1-lirongqing@baidu.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/cq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 7495e20c3dec..55ffd7d4de51 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -1316,7 +1316,7 @@ static int copy_resize_cqes(struct mlx5_ib_cq *cq) while (get_cqe_opcode(scqe64) != MLX5_CQE_RESIZE_CQ) { dcqe = mlx5_frag_buf_get_wqe(&cq->resize_buf->fbc, - (i + 1) & cq->resize_buf->nent); + (i + 1) & (cq->resize_buf->nent - 1)); dcqe64 = dsize == 64 ? dcqe : dcqe + 64; sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent); memcpy(dcqe, scqe, dsize); From bb75e5580b15cd23ededf15683a5ad67d7b61aab Mon Sep 17 00:00:00 2001 From: Serhat Kumral Date: Sun, 18 Oct 2026 17:49:07 +0300 Subject: [PATCH 152/153] RDMA/mlx5: Validate priority in STEERING_ANCHOR_CREATE handler In MLX5_IB_METHOD_STEERING_ANCHOR_CREATE, the user-supplied u16 priority is passed directly to _get_flow_table() without an upper bound check. This can lead to out-of-bounds array accesses on dev->flow_db priority tables. Check that priority does not exceed MLX5_IB_FLOW_LAST_PRIO before calling _get_flow_table(), matching the existing validation in raw_fs_rule_add() and mlx5_ib_create_flow(). Fixes: 0c6ab0ca9a66 ("RDMA/mlx5: Expose steering anchor to userspace") Signed-off-by: Serhat Kumral Link: https://patch.msgid.link/20261018144907.20123-1-serhatkumral1@gmail.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/mlx5/fs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c index e63fdb4bc3e3..bc2c15c7172b 100644 --- a/drivers/infiniband/hw/mlx5/fs.c +++ b/drivers/infiniband/hw/mlx5/fs.c @@ -3014,6 +3014,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( if (err) return err; + if (priority > MLX5_IB_FLOW_LAST_PRIO) + return -EINVAL; + obj = kzalloc_obj(*obj); if (!obj) return -ENOMEM; From c3e8e9451e6f1b69f685b75e0fbf14f3d0fb2539 Mon Sep 17 00:00:00 2001 From: Sunny Tiwari Date: Sat, 17 Oct 2026 05:28:56 +0000 Subject: [PATCH 153/153] RDMA/irdma: Bound in-flight CQP ops across deferred completions On IRDMA_GEN_3+ devices, CQP operations can complete in two stages: Stage-1 on the CCQ acknowledges receipt and moves an entry from cqp->ooo_avail to cqp->ooo_pnd, while Stage-2 signals final completion via an AEQ event or a subsequent CCQ CQE. In irdma_sc_ccq_get_cqe_info(), Stage-1 processing advances cqp->sq_ring.tail so the SQ descriptor slot can be reused once consumed by hardware, whereas cqp->completed_ops is only incremented on Stage-2. Because irdma_cqp_ring_full() only checks cqp->sq_ring, a burst of backlogged commands in dev->cqp_cmd_head allows irdma_process_bh() to submit more concurrent in-flight CQP commands than the maximum supported limit (cqp->sq_size - 1) while earlier commands await Stage-2. This causes CQP completions to be lost and synchronous waiters to time out with -ETIMEDOUT (-110). Fix this by: - Updating irdma_cqp_ring_full() to bound total in-flight CQP operations (cqp->requested_ops - cqp->completed_ops) to (cqp->sq_size - 1). - Calling irdma_process_bh() at the end of irdma_process_ae_def_cmpl() so backlogged commands are drained after Stage-2 AEQ completions increment cqp->completed_ops and free in-flight capacity. Fixes: c7db0abe5f2b ("RDMA/irdma: Add GEN3 CQP support with deferred completions") Assisted-by: Gemini Signed-off-by: Sunny Tiwari Link: https://patch.msgid.link/20261017052856.3570063-1-sunnytiwari@google.com Reviewed-by: Jacob Moroni Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/irdma/ctrl.c | 6 +++++- drivers/infiniband/hw/irdma/hw.c | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c index b62bf4d11829..1c2ad5773b9f 100644 --- a/drivers/infiniband/hw/irdma/ctrl.c +++ b/drivers/infiniband/hw/irdma/ctrl.c @@ -5323,7 +5323,11 @@ int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, */ static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp) { - return IRDMA_RING_FULL_ERR(cqp->sq_ring); + u64 in_flight = cqp->requested_ops - + atomic64_read(&cqp->completed_ops); + + return IRDMA_RING_FULL_ERR(cqp->sq_ring) || + in_flight >= (cqp->sq_size - 1); } /** diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index e28f5f4876cd..fbb8fb47d0d1 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -265,6 +265,7 @@ static void irdma_process_ae_def_cmpl(struct irdma_pci_f *rf, irdma_sc_cqp_def_cmpl_ae_handler(&rf->sc_dev, info, false, &scratch, &sw_def_info); } + irdma_process_bh(&rf->sc_dev); } /**