mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
RDMA/ionic: support firmware-assigned CQ IDs
When the LIF advertises qid allocation for CQ, let firmware return the completion queue ID instead of allocating it in the driver bitmap. Split CQ create/destroy into distinct phases so cq_tbl is updated only after the ID is known and removed before the destroy admin command returns the ID to firmware, avoiding a race where a concurrent create reusing the same cqid could have its xarray entry erased by a stale destroy. Signed-off-by: Abhijit Gangurde <abhijit.gangurde@amd.com> Link: https://patch.msgid.link/20260930092111.3176738-3-abhijit.gangurde@amd.com Signed-off-by: Leon Romanovsky <leon@kernel.org>
This commit is contained in:
Abhijit Gangurde
Leon Romanovsky
parent
6de34bfb56
commit
1900af7931
|
|
@ -489,6 +489,49 @@ static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev,
|
|||
return ionic_rdma_devcmd(dev, &admin);
|
||||
}
|
||||
|
||||
static int ionic_rdma_cq_devcmd(struct ionic_vcq *vcq,
|
||||
struct ionic_queue *q,
|
||||
u32 *qid, u32 cid, u16 opcode,
|
||||
u8 udma_idx)
|
||||
{
|
||||
struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device);
|
||||
struct ionic_admin_ctx admin = {
|
||||
.work = COMPLETION_INITIALIZER_ONSTACK(admin.work),
|
||||
.cmd.rdma_queue = {
|
||||
.opcode = opcode,
|
||||
.lif_index = cpu_to_le16(dev->lif_cfg.lif_index),
|
||||
.qid_ver = cpu_to_le32(*qid),
|
||||
.cid = cpu_to_le32(cid),
|
||||
.dbid = cpu_to_le16(dev->lif_cfg.dbid),
|
||||
.depth_log2 = q->depth_log2,
|
||||
.stride_log2 = q->stride_log2,
|
||||
.dma_addr = cpu_to_le64(q->dma),
|
||||
},
|
||||
};
|
||||
int rc;
|
||||
|
||||
if (ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) {
|
||||
struct ionic_admin_create_cq_resp resp_buf;
|
||||
|
||||
/* Indicates use of CQ create V2 response format. */
|
||||
admin.cmd.rdma_queue.qid_ver = cpu_to_le32(IONIC_CREATE_CQ_CMD_V2_MAGIC);
|
||||
admin.cmd.rdma_queue.udma_idx = udma_idx;
|
||||
|
||||
rc = ionic_rdma_devcmd(dev, &admin);
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
memcpy(&resp_buf, admin.comp.comp.cmd_data, sizeof(resp_buf));
|
||||
*qid = le32_to_cpu(resp_buf.id);
|
||||
} else {
|
||||
rc = ionic_rdma_devcmd(dev, &admin);
|
||||
if (rc)
|
||||
return rc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context)
|
||||
{
|
||||
struct ionic_aq *aq = cq_context;
|
||||
|
|
@ -533,12 +576,16 @@ static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev,
|
|||
cq = &vcq->cq[0];
|
||||
|
||||
rc = ionic_create_cq_common(vcq, &buf, &attr, NULL, NULL,
|
||||
NULL, NULL, 0);
|
||||
NULL, 0);
|
||||
if (rc)
|
||||
goto err_init;
|
||||
|
||||
rc = ionic_rdma_queue_devcmd(dev, &cq->q, cq->cqid, cq->eqid,
|
||||
IONIC_CMD_RDMA_CREATE_CQ);
|
||||
rc = ionic_rdma_cq_devcmd(vcq, &cq->q, &cq->cqid, cq->eqid,
|
||||
IONIC_CMD_RDMA_CREATE_CQ, 0);
|
||||
if (rc)
|
||||
goto err_cmd;
|
||||
|
||||
rc = ionic_post_create_cq_cmd(cq, NULL, NULL);
|
||||
if (rc)
|
||||
goto err_cmd;
|
||||
|
||||
|
|
@ -552,6 +599,14 @@ static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev,
|
|||
return ERR_PTR(rc);
|
||||
}
|
||||
|
||||
static void ionic_destroy_rdma_admincq(struct ionic_ibdev *dev,
|
||||
struct ionic_vcq *vcq)
|
||||
{
|
||||
ionic_pre_destroy_cq_cmd(dev, &vcq->cq[0]);
|
||||
ionic_destroy_cq_common(dev, &vcq->cq[0]);
|
||||
kfree(vcq);
|
||||
}
|
||||
|
||||
static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev,
|
||||
u32 aqid, u32 cqid)
|
||||
{
|
||||
|
|
@ -1153,8 +1208,7 @@ int ionic_create_rdma_admin(struct ionic_ibdev *dev)
|
|||
vcq->cq[0].cqid);
|
||||
if (IS_ERR(aq)) {
|
||||
/* Clean up the dangling CQ */
|
||||
ionic_destroy_cq_common(dev, &vcq->cq[0]);
|
||||
kfree(vcq);
|
||||
ionic_destroy_rdma_admincq(dev, vcq);
|
||||
|
||||
rc = PTR_ERR(aq);
|
||||
|
||||
|
|
@ -1207,10 +1261,8 @@ void ionic_destroy_rdma_admin(struct ionic_ibdev *dev)
|
|||
cancel_work_sync(&aq->work);
|
||||
|
||||
__ionic_destroy_rdma_adminq(dev, aq);
|
||||
if (vcq) {
|
||||
ionic_destroy_cq_common(dev, &vcq->cq[0]);
|
||||
kfree(vcq);
|
||||
}
|
||||
if (vcq)
|
||||
ionic_destroy_rdma_admincq(dev, vcq);
|
||||
}
|
||||
|
||||
kfree(dev->aq_vec);
|
||||
|
|
|
|||
|
|
@ -79,12 +79,10 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
|
|||
struct ionic_ctx *ctx,
|
||||
struct ib_udata *udata,
|
||||
struct ionic_qdesc *req_cq,
|
||||
__u32 *resp_cqid,
|
||||
int udma_idx)
|
||||
{
|
||||
struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device);
|
||||
struct ionic_cq *cq = &vcq->cq[udma_idx];
|
||||
void *entry;
|
||||
int rc;
|
||||
|
||||
cq->vcq = vcq;
|
||||
|
|
@ -94,9 +92,11 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
|
|||
goto err_args;
|
||||
}
|
||||
|
||||
rc = ionic_get_cqid(dev, &cq->cqid, udma_idx);
|
||||
if (rc)
|
||||
goto err_args;
|
||||
if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) {
|
||||
rc = ionic_get_cqid(dev, &cq->cqid, udma_idx);
|
||||
if (rc)
|
||||
goto err_args;
|
||||
}
|
||||
|
||||
cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx);
|
||||
|
||||
|
|
@ -123,8 +123,6 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
|
|||
cq->q.mask = req_cq->mask;
|
||||
cq->q.depth_log2 = req_cq->depth_log2;
|
||||
cq->q.stride_log2 = req_cq->stride_log2;
|
||||
|
||||
*resp_cqid = cq->cqid;
|
||||
} else {
|
||||
rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev,
|
||||
attr->cqe + IONIC_CQ_GRACE,
|
||||
|
|
@ -132,7 +130,6 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
|
|||
if (rc)
|
||||
goto err_q_init;
|
||||
|
||||
ionic_queue_dbell_init(&cq->q, cq->cqid);
|
||||
cq->color = true;
|
||||
cq->credit = cq->q.mask;
|
||||
}
|
||||
|
|
@ -144,20 +141,8 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
|
|||
init_completion(&cq->cq_rel_comp);
|
||||
kref_init(&cq->cq_kref);
|
||||
|
||||
entry = xa_store_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL);
|
||||
if (entry) {
|
||||
if (!xa_is_err(entry))
|
||||
rc = -EINVAL;
|
||||
else
|
||||
rc = xa_err(entry);
|
||||
|
||||
goto err_xa;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
err_xa:
|
||||
ionic_pgtbl_unbuf(dev, buf);
|
||||
err_pgtbl_init:
|
||||
if (!udata)
|
||||
ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);
|
||||
|
|
@ -165,29 +150,52 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
|
|||
if (cq->umem)
|
||||
ib_umem_release(cq->umem);
|
||||
err_qdesc:
|
||||
ionic_put_cqid(dev, cq->cqid);
|
||||
if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ))
|
||||
ionic_put_cqid(dev, cq->cqid);
|
||||
err_args:
|
||||
cq->vcq = NULL;
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
int ionic_post_create_cq_cmd(struct ionic_cq *cq,
|
||||
struct ib_udata *udata,
|
||||
__u32 *resp_cqid)
|
||||
{
|
||||
struct ionic_ibdev *dev = to_ionic_ibdev(cq->vcq->ibcq.device);
|
||||
|
||||
if (udata && resp_cqid)
|
||||
*resp_cqid = cq->cqid;
|
||||
else
|
||||
ionic_queue_dbell_init(&cq->q, cq->cqid);
|
||||
|
||||
return xa_insert_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL);
|
||||
}
|
||||
|
||||
void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq)
|
||||
{
|
||||
if (!cq->vcq)
|
||||
return;
|
||||
|
||||
if (!xa_erase_irq(&dev->cq_tbl, cq->cqid))
|
||||
return;
|
||||
|
||||
kref_put(&cq->cq_kref, ionic_cq_complete);
|
||||
wait_for_completion(&cq->cq_rel_comp);
|
||||
}
|
||||
|
||||
void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq)
|
||||
{
|
||||
if (!cq->vcq)
|
||||
return;
|
||||
|
||||
xa_erase_irq(&dev->cq_tbl, cq->cqid);
|
||||
|
||||
kref_put(&cq->cq_kref, ionic_cq_complete);
|
||||
wait_for_completion(&cq->cq_rel_comp);
|
||||
|
||||
if (cq->umem)
|
||||
ib_umem_release(cq->umem);
|
||||
else
|
||||
ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);
|
||||
|
||||
ionic_put_cqid(dev, cq->cqid);
|
||||
if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ))
|
||||
ionic_put_cqid(dev, cq->cqid);
|
||||
|
||||
cq->vcq = NULL;
|
||||
}
|
||||
|
|
@ -1223,8 +1231,11 @@ int ionic_dealloc_mw(struct ib_mw *ibmw)
|
|||
static int ionic_create_cq_cmd(struct ionic_ibdev *dev,
|
||||
struct ionic_ctx *ctx,
|
||||
struct ionic_cq *cq,
|
||||
struct ionic_tbl_buf *buf)
|
||||
struct ionic_tbl_buf *buf,
|
||||
int udma_idx,
|
||||
int *out_udma_idx)
|
||||
{
|
||||
struct ionic_admin_create_cq_resp *resp_buf;
|
||||
const u16 dbid = ionic_ctx_dbid(dev, ctx);
|
||||
struct ionic_admin_wr wr = {
|
||||
.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
|
||||
|
|
@ -1244,13 +1255,57 @@ static int ionic_create_cq_cmd(struct ionic_ibdev *dev,
|
|||
}
|
||||
}
|
||||
};
|
||||
dma_addr_t resp_buf_dma;
|
||||
int rc;
|
||||
|
||||
if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ)
|
||||
return -EBADRQC;
|
||||
|
||||
ionic_admin_post(dev, &wr);
|
||||
if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) {
|
||||
ionic_admin_post(dev, &wr);
|
||||
return ionic_admin_wait(dev, &wr, 0);
|
||||
}
|
||||
|
||||
return ionic_admin_wait(dev, &wr, 0);
|
||||
resp_buf = kzalloc_obj(*resp_buf);
|
||||
if (!resp_buf)
|
||||
return -ENOMEM;
|
||||
|
||||
resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf,
|
||||
sizeof(*resp_buf),
|
||||
DMA_FROM_DEVICE);
|
||||
|
||||
rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma);
|
||||
if (rc) {
|
||||
rc = -ENOMEM;
|
||||
goto err_dma;
|
||||
}
|
||||
|
||||
wr.wqe.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V2_LEN);
|
||||
wr.wqe.cmd.create_cq.udma_idx = udma_idx;
|
||||
wr.wqe.cmd.create_cq.resp_dma_addr = cpu_to_le64(resp_buf_dma);
|
||||
wr.wqe.cmd.create_cq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN);
|
||||
|
||||
ionic_admin_post(dev, &wr);
|
||||
rc = ionic_admin_wait(dev, &wr, 0);
|
||||
|
||||
dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf),
|
||||
DMA_FROM_DEVICE);
|
||||
if (rc)
|
||||
goto err_dma;
|
||||
|
||||
if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN) {
|
||||
rc = -EOPNOTSUPP;
|
||||
goto err_dma;
|
||||
}
|
||||
|
||||
cq->cqid = le32_to_cpu(resp_buf->id);
|
||||
if (out_udma_idx)
|
||||
*out_udma_idx = resp_buf->udma_idx;
|
||||
|
||||
err_dma:
|
||||
kfree(resp_buf);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid)
|
||||
|
|
@ -1309,16 +1364,21 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
|
|||
|
||||
rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata,
|
||||
&req.cq[udma_idx],
|
||||
&resp.cqid[udma_idx],
|
||||
udma_idx);
|
||||
if (rc)
|
||||
goto err_init;
|
||||
|
||||
rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf);
|
||||
rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf,
|
||||
udma_idx, NULL);
|
||||
if (rc)
|
||||
goto err_cmd;
|
||||
|
||||
ionic_pgtbl_unbuf(dev, &buf);
|
||||
|
||||
rc = ionic_post_create_cq_cmd(&vcq->cq[udma_idx], udata,
|
||||
&resp.cqid[udma_idx]);
|
||||
if (rc)
|
||||
goto err_post;
|
||||
}
|
||||
|
||||
vcq->ibcq.cqe = attr->cqe;
|
||||
|
|
@ -1338,6 +1398,8 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
|
|||
--udma_idx;
|
||||
if (!(vcq->udma_mask & BIT(udma_idx)))
|
||||
continue;
|
||||
ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]);
|
||||
err_post:
|
||||
ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid);
|
||||
err_cmd:
|
||||
ionic_pgtbl_unbuf(dev, &buf);
|
||||
|
|
@ -1365,6 +1427,13 @@ int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
|
|||
if (!(vcq->udma_mask & BIT(udma_idx)))
|
||||
continue;
|
||||
|
||||
/*
|
||||
* Untrack the CQ before releasing its hardware ID below, so a
|
||||
* concurrent create that gets the same ID reused by firmware
|
||||
* cannot have its fresh XArray entry erased by this destroy.
|
||||
*/
|
||||
ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]);
|
||||
|
||||
rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid);
|
||||
if (rc_tmp) {
|
||||
if (!rc)
|
||||
|
|
|
|||
|
|
@ -746,11 +746,22 @@ struct ionic_admin_create_cq {
|
|||
__le32 map_count;
|
||||
__le64 dma_addr;
|
||||
__le16 dbid_flags;
|
||||
__u8 udma_idx;
|
||||
__le32 resp_buf_len;
|
||||
__le64 resp_dma_addr;
|
||||
} __packed;
|
||||
|
||||
#define IONIC_ADMIN_CREATE_CQ_IN_V1_LEN 34
|
||||
static_assert(sizeof(struct ionic_admin_create_cq) ==
|
||||
IONIC_ADMIN_CREATE_CQ_IN_V1_LEN);
|
||||
#define IONIC_ADMIN_CREATE_CQ_IN_V2_LEN 47
|
||||
static_assert(sizeof(struct ionic_admin_create_cq) == IONIC_ADMIN_CREATE_CQ_IN_V2_LEN);
|
||||
|
||||
struct ionic_admin_create_cq_resp {
|
||||
__le32 id;
|
||||
__u8 udma_idx;
|
||||
} __packed;
|
||||
|
||||
#define IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN 5
|
||||
static_assert(sizeof(struct ionic_admin_create_cq_resp) == IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN);
|
||||
|
||||
struct ionic_admin_destroy_cq {
|
||||
__le32 cq_id;
|
||||
|
|
|
|||
|
|
@ -438,6 +438,12 @@ static inline void ionic_cq_complete(struct kref *kref)
|
|||
complete(&cq->cq_rel_comp);
|
||||
}
|
||||
|
||||
static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev,
|
||||
enum ionic_lif_rdma_alloc_qid qtype)
|
||||
{
|
||||
return dev->lif_cfg.alloc_qid_cap & qtype;
|
||||
}
|
||||
|
||||
/* ionic_admin.c */
|
||||
extern struct workqueue_struct *ionic_evt_workq;
|
||||
void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);
|
||||
|
|
@ -457,8 +463,11 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
|
|||
struct ionic_ctx *ctx,
|
||||
struct ib_udata *udata,
|
||||
struct ionic_qdesc *req_cq,
|
||||
__u32 *resp_cqid,
|
||||
int udma_idx);
|
||||
int ionic_post_create_cq_cmd(struct ionic_cq *cq,
|
||||
struct ib_udata *udata,
|
||||
__u32 *resp_cqid);
|
||||
void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq);
|
||||
void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);
|
||||
void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);
|
||||
void ionic_notify_flush_cq(struct ionic_cq *cq);
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg)
|
|||
!!(lif->qtype_info[IONIC_QTYPE_TXQ].features & IONIC_QIDENT_F_EXPDB);
|
||||
cfg->rq_expdb =
|
||||
!!(lif->qtype_info[IONIC_QTYPE_RXQ].features & IONIC_QIDENT_F_EXPDB);
|
||||
cfg->alloc_qid_cap = ident->rdma.alloc_qid_cap;
|
||||
}
|
||||
|
||||
struct net_device *ionic_lif_netdev(struct ionic_lif *lif)
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ struct ionic_lif_cfg {
|
|||
bool rq_expdb;
|
||||
u8 expdb_mask;
|
||||
u8 rcq_sign_bit;
|
||||
u8 alloc_qid_cap;
|
||||
};
|
||||
|
||||
void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg);
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user