github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

RDMA/umem: Reuse ib_umem_get_cq_buf_or_va() for VA-only CQ pinning

Browse Source 
Several drivers pin their CQ ring buffer via ib_umem_get_va() rather
than the CQ-specific helper. Switch them to ib_umem_get_cq_buf_or_va()
so the subsequent DMA_FROM_DEVICE change covers all CQ paths at once.

For shared helpers used by both CQ and non-CQ callers (qedr, mana,
erdma, hns) a new is_cq parameter selects the appropriate pinning
function; non-CQ callers pass false.

vmw_pvrdma's CQ is excluded: it embeds a ring-state header that the
driver CPU-writes after polling, making it genuinely bidirectional.

This is a pure refactor with no functional change.

Signed-off-by: Yishai Hadas <yishaih@nvidia.com>
Link: https://patch.msgid.link/20260915140933.40580-5-yishaih@nvidia.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>
This commit is contained in:
Yishai Hadas 2026-09-15 17:09:22 +03:00 committed by Leon Romanovsky
parent 89abac4b60
commit b577999f6f
18 changed files with 83 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
drivers/infiniband/hw/bnxt_re/ib_verbs.c
View File

@ -3749,12 +3749,13 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe,
if (rc)
goto fail;
cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va,
entries * sizeof(struct cq_base),
IB_ACCESS_LOCAL_WRITE);
cq->resize_umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, NULL,
req.cq_va,
entries * sizeof(struct cq_base),
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->resize_umem)) {
rc = PTR_ERR(cq->resize_umem);
ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n",
ibdev_err(&rdev->ibdev, "%s: ib_umem_get_cq_buf_or_va failed! rc = %pe\n",
__func__, cq->resize_umem);
cq->resize_umem = NULL;
goto fail;

11
drivers/infiniband/hw/erdma/erdma_verbs.c
View File

@ -788,8 +788,14 @@ static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
switch (mem->type) {
case ERDMA_UMEM:
mem->umem = ib_umem_get_va(&dev->ibdev, attr->start, attr->len,
attr->access);
if (attr->flags & ERDMA_MEM_FLAG_CQ_BUF)
mem->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
attr->start,
attr->len,
attr->access);
else
mem->umem = ib_umem_get_va(&dev->ibdev, attr->start,
attr->len, attr->access);
if (IS_ERR(mem->umem)) {
ret = PTR_ERR(mem->umem);
return ret;
@ -2000,6 +2006,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
attr.len = ureq->qbuf_len;
attr.req_page_size = SZ_64M - SZ_4K;
attr.access = IB_ACCESS_LOCAL_WRITE;
attr.flags = ERDMA_MEM_FLAG_CQ_BUF;
ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr);
if (ret)
return ret;

1
drivers/infiniband/hw/erdma/erdma_verbs.h
View File

@ -128,6 +128,7 @@ struct erdma_kmem {
enum erdma_mem_flags {
ERDMA_MEM_FLAG_MR_BUF = (1 << 0),
ERDMA_MEM_FLAG_CQ_BUF = (1 << 1),
};
struct erdma_mem_init_attr {

2
drivers/infiniband/hw/hns/hns_roce_cq.c
View File

@ -265,7 +265,7 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,
hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT,
udata, addr);
udata, addr, true);
if (ret)
ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret);

2
drivers/infiniband/hw/hns/hns_roce_device.h
View File

@ -1232,7 +1232,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
struct hns_roce_buf_attr *buf_attr,
unsigned int page_shift, struct ib_udata *udata,
unsigned long user_addr);
unsigned long user_addr, bool is_cq);
void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
struct hns_roce_mtr *mtr);
int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,

2
drivers/infiniband/hw/hns/hns_roce_hw_v2.c
View File

@ -6940,7 +6940,7 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,
0);
0, false);
if (err)
dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err);

22
drivers/infiniband/hw/hns/hns_roce_mr.c
View File

@ -112,7 +112,7 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr,
hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT,
udata, start);
udata, start, false);
if (err) {
ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err);
return err;
@ -586,7 +586,8 @@ static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
struct hns_roce_buf_attr *buf_attr,
struct ib_udata *udata, unsigned long user_addr)
struct ib_udata *udata, unsigned long user_addr,
bool is_cq)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
size_t total_size;
@ -595,8 +596,14 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
if (udata) {
mtr->kmem = NULL;
mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
buf_attr->user_access);
if (is_cq)
mtr->umem = ib_umem_get_cq_buf_or_va(ibdev, NULL,
user_addr,
total_size,
buf_attr->user_access);
else
mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
buf_attr->user_access);
if (IS_ERR(mtr->umem)) {
ibdev_err(ibdev, "failed to get umem, ret = %pe.\n",
mtr->umem);
@ -1035,11 +1042,13 @@ static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
* @ba_page_shift: page shift for multi-hop base address table
* @udata: user space context, if it's NULL, means kernel space
* @user_addr: userspace virtual address to start at
* @is_cq: true when @mtr backs a CQ buffer, which the device writes
* completion entries into
*/
int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
struct hns_roce_buf_attr *buf_attr,
unsigned int ba_page_shift, struct ib_udata *udata,
unsigned long user_addr)
unsigned long user_addr, bool is_cq)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
int ret;
@ -1052,7 +1061,8 @@ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
mtr->umem = NULL;
mtr->kmem = NULL;
} else {
ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr);
ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr,
is_cq);
if (ret) {
ibdev_err(ibdev,
"failed to alloc mtr bufs, ret = %d.\n", ret);

2
drivers/infiniband/hw/hns/hns_roce_qp.c
View File

@ -808,7 +808,7 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
}
ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
udata, addr);
udata, addr, false);
if (ret) {
ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
goto err_inline;

4
drivers/infiniband/hw/hns/hns_roce_srq.c
View File

@ -180,7 +180,7 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr,
hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT,
udata, addr);
udata, addr, false);
if (ret) {
ibdev_err(ibdev,
"failed to alloc SRQ idx mtr, ret = %d.\n", ret);
@ -235,7 +235,7 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev,
ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr,
hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT,
udata, addr);
udata, addr, false);
if (ret)
ibdev_err(ibdev,
"failed to alloc SRQ buf mtr, ret = %d.\n", ret);

5
drivers/infiniband/hw/ionic/ionic_controlpath.c
View File

@ -110,8 +110,9 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
if (rc)
goto err_qdesc;
cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr,
req_cq->size, IB_ACCESS_LOCAL_WRITE);
cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
req_cq->addr, req_cq->size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->umem)) {
rc = PTR_ERR(cq->umem);
goto err_qdesc;

2
drivers/infiniband/hw/mana/cq.c
View File

@ -40,7 +40,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
cq->cqe = attr->cqe;
err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
&cq->queue);
&cq->queue, true);
if (err) {
ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
return err;

9
drivers/infiniband/hw/mana/main.c
View File

@ -295,7 +295,7 @@ int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_qu
}
int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
struct mana_ib_queue *queue)
struct mana_ib_queue *queue, bool is_cq)
{
struct ib_umem *umem;
int err;
@ -304,7 +304,12 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
queue->id = INVALID_QUEUE_ID;
queue->gdma_region = GDMA_INVALID_DMA_REGION;
umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
if (is_cq)
umem = ib_umem_get_cq_buf_or_va(&mdev->ib_dev, NULL, addr,
size, IB_ACCESS_LOCAL_WRITE);
else
umem = ib_umem_get_va(&mdev->ib_dev, addr, size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem);
return PTR_ERR(umem);

2
drivers/infiniband/hw/mana/mana_ib.h
View File

@ -731,7 +731,7 @@ int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type,
struct mana_ib_queue *queue);
int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
struct mana_ib_queue *queue);
struct mana_ib_queue *queue, bool is_cq);
void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,

9
drivers/infiniband/hw/mana/qp.c
View File

@ -340,7 +340,8 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",
ucmd.sq_buf_addr, ucmd.port);
err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq);
err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq,
false);
if (err) {
ibdev_dbg(&mdev->ib_dev,
"Failed to create queue for create qp-raw, err %d\n", err);
@ -579,7 +580,7 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE) {
flags &= ~MANA_RC_FLAG_NO_MMQ;
err = mana_ib_create_queue(mdev, ucmd.mmq_buf, ucmd.mmq_size,
&qp->rc_qp.queues[i]);
&qp->rc_qp.queues[i], false);
if (err)
goto destroy_queues;
} else {
@ -589,7 +590,7 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
continue;
}
err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],
&qp->rc_qp.queues[i]);
&qp->rc_qp.queues[i], false);
if (err)
goto destroy_queues;
j++;
@ -663,7 +664,7 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; ++i) {
err = mana_ib_create_queue(mdev, ucmd.queue_buf[i], ucmd.queue_size[i],
&qp->uc_qp.queues[i]);
&qp->uc_qp.queues[i], false);
if (err)
goto destroy_queues;
}

3
drivers/infiniband/hw/mana/wq.c
View File

@ -25,7 +25,8 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr);
err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue);
err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue,
false);
if (err) {
ibdev_dbg(&mdev->ib_dev,
"Failed to create queue for create wq, %d\n", err);

14
drivers/infiniband/hw/mlx4/cq.c
View File

@ -185,9 +185,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
goto err_umem;
}
} else {
cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
cq->umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL,
ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->umem)) {
err = PTR_ERR(cq->umem);
goto err_cq;
@ -354,9 +355,10 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq
if (!cq->resize_buf)
return -ENOMEM;
cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
cq->resize_umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL,
ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->resize_umem)) {
err = PTR_ERR(cq->resize_umem);
goto err_buf;

6
drivers/infiniband/hw/mlx5/cq.c
View File

@ -1246,9 +1246,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1)
return -EINVAL;
umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
(size_t)ucmd.cqe_size * entries,
IB_ACCESS_LOCAL_WRITE);
umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, ucmd.buf_addr,
(size_t)ucmd.cqe_size * entries,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
err = PTR_ERR(umem);
return err;

23
drivers/infiniband/hw/qedr/verbs.c
View File

@ -775,16 +775,23 @@ static inline int qedr_init_user_queue(struct ib_udata *udata,
struct qedr_userq *q, u64 buf_addr,
size_t buf_len, bool requires_db_rec,
int access,
int alloc_and_init)
int alloc_and_init, bool is_cq)
{
u32 fw_pages;
int rc;
q->buf_addr = buf_addr;
q->buf_len = buf_len;
q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, access);
if (is_cq)
q->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
q->buf_addr, q->buf_len,
access);
else
q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len,
access);
if (IS_ERR(q->umem)) {
DP_ERR(dev, "create user queue: failed ib_umem_get_va, got %ld\n",
DP_ERR(dev, "create user queue: failed %s, got %ld\n",
is_cq ? "ib_umem_get_cq_buf_or_va" : "ib_umem_get_va",
PTR_ERR(q->umem));
return PTR_ERR(q->umem);
}
@ -946,7 +953,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
ureq.len, true, IB_ACCESS_LOCAL_WRITE,
1);
1, true);
if (rc)
goto err0;
@ -1438,7 +1445,7 @@ static int qedr_init_srq_user_params(struct ib_udata *udata,
int rc;
rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
ureq->srq_len, false, access, 1);
ureq->srq_len, false, access, 1, false);
if (rc)
return rc;
@ -1831,7 +1838,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev,
if (qedr_qp_has_sq(qp)) {
/* SQ - read access only (0) */
rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
ureq.sq_len, true, 0, alloc_and_init);
ureq.sq_len, true, 0, alloc_and_init,
false);
if (rc)
return rc;
}
@ -1839,7 +1847,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev,
if (qedr_qp_has_rq(qp)) {
/* RQ - read access only (0) */
rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
ureq.rq_len, true, 0, alloc_and_init);
ureq.rq_len, true, 0, alloc_and_init,
false);
if (rc) {
ib_umem_release(qp->usq.umem);
qp->usq.umem = NULL;