diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 9184b2b95834..0e44c7fcf5fe 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -2169,7 +2169,7 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); if (uctx) - free_page((unsigned long)srq->uctx_srq_page); + rdma_user_mmap_entry_remove(&srq->toggle_entry->rdma_entry); } ib_umem_release(srq->umem); atomic_dec(&rdev->stats.res.srq_count); @@ -2282,10 +2282,17 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, rc = -ENOMEM; goto fail_destroy_srq; } + srq->toggle_entry = bnxt_re_mmap_entry_insert(uctx, (u64)srq->uctx_srq_page, + BNXT_RE_MMAP_TOGGLE_PAGE, + NULL); + if (!srq->toggle_entry) { + rc = -ENOMEM; + goto fail_free_srq_page; + } if (xa_is_err(xa_store(&uctx->srq_xa, srq->qplib_srq.id, ib_srq->uobject, GFP_KERNEL))) { rc = -ENOMEM; - goto fail_free_srq_page; + goto fail_remove_toggle_entry; } resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT; } @@ -2303,10 +2310,13 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, fail_respond: if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) { xa_erase(&uctx->srq_xa, srq->qplib_srq.id); - free_page((unsigned long)srq->uctx_srq_page); + goto fail_remove_toggle_entry; } bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq); goto fail; +fail_remove_toggle_entry: + rdma_user_mmap_entry_remove(&srq->toggle_entry->rdma_entry); + goto fail_destroy_srq; fail_free_srq_page: free_page((unsigned long)srq->uctx_srq_page); fail_destroy_srq: @@ -3510,7 +3520,7 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx); if (uctx) - free_page((unsigned long)cq->uctx_cq_page); + rdma_user_mmap_entry_remove(&cq->toggle_entry->rdma_entry); } bnxt_re_put_nq(rdev, nq); @@ -3591,10 +3601,16 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att rc = -ENOMEM; goto destroy_cq; } + cq->toggle_entry = bnxt_re_mmap_entry_insert(uctx, (u64)cq->uctx_cq_page, + BNXT_RE_MMAP_TOGGLE_PAGE, NULL); + if (!cq->toggle_entry) { + rc = -ENOMEM; + goto free_cq_page; + } if (xa_is_err(xa_store(&uctx->cq_xa, cq->qplib_cq.id, ibcq->uobject, GFP_KERNEL))) { rc = -ENOMEM; - goto free_cq_page; + goto remove_toggle_entry; } resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT; } @@ -3610,6 +3626,10 @@ int bnxt_re_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *att free_mem: if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) xa_erase(&uctx->cq_xa, cq->qplib_cq.id); +remove_toggle_entry: + if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) + rdma_user_mmap_entry_remove(&cq->toggle_entry->rdma_entry); + goto destroy_cq; free_cq_page: free_page((unsigned long)cq->uctx_cq_page); destroy_cq: @@ -5060,6 +5080,16 @@ void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry) bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry, rdma_entry); + /* + * For toggle pages the kernel VA was stored directly in mem_offset + * at creation time (bnxt_re_create_user_cq / bnxt_re_create_srq). + * Free it here — this is the only place it is freed, ensuring the + * page outlives every concurrent bnxt_re_mmap() call that may have + * incremented the entry's reference count. + */ + if (bnxt_entry->mmap_flag == BNXT_RE_MMAP_TOGGLE_PAGE) + free_page((unsigned long)bnxt_entry->mem_offset); + if (bnxt_entry->dpi_valid) bnxt_qplib_free_uc_dpi(&bnxt_entry->uctx->rdev->qplib_res, &bnxt_entry->dpi); diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 4c78c183784b..b7b33f6acf91 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -80,6 +80,7 @@ struct bnxt_re_srq { struct ib_umem *umem; spinlock_t lock; /* protect srq */ void *uctx_srq_page; + struct bnxt_re_user_mmap_entry *toggle_entry; }; struct bnxt_re_qp { @@ -114,6 +115,7 @@ struct bnxt_re_cq { struct ib_umem *resize_umem; int resize_cqe; void *uctx_cq_page; + struct bnxt_re_user_mmap_entry *toggle_entry; }; struct bnxt_re_mr { diff --git a/drivers/infiniband/hw/bnxt_re/uapi.c b/drivers/infiniband/hw/bnxt_re/uapi.c index c5e4e6e47b5f..97bc0e755511 100644 --- a/drivers/infiniband/hw/bnxt_re/uapi.c +++ b/drivers/infiniband/hw/bnxt_re/uapi.c @@ -213,19 +213,23 @@ DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV, &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV)); /* Toggle MEM */ +struct bnxt_re_toggle_mem { + struct bnxt_re_user_mmap_entry *toggle_entry; + u64 mmap_offset; +}; + static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bundle *attrs) { struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_TOGGLE_MEM_HANDLE); - enum bnxt_re_mmap_flag mmap_flag = BNXT_RE_MMAP_TOGGLE_PAGE; + struct bnxt_re_user_mmap_entry *toggle_entry = NULL; enum bnxt_re_get_toggle_mem_type res_type; - struct bnxt_re_user_mmap_entry *entry; + struct bnxt_re_toggle_mem *tmem; struct ib_uobject *res_uobj; struct bnxt_re_ucontext *uctx; struct ib_ucontext *ib_uctx; u32 length = PAGE_SIZE; - u64 mem_offset; + u64 mmap_offset = 0; u32 offset = 0; - u64 addr = 0; u32 res_id; int err; @@ -243,6 +247,10 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund return err; /* + * Hold xa_lock across xa_load + kref_get so that a concurrent + * bnxt_re_destroy_cq/srq cannot call __xa_erase and remove the + * toggle_entry between our load and our reference on it. + * * bnxt_re_create_cq/srq() publishes the uobject into cq_xa/srq_xa * before returning to the uverbs core, but the core only sets * uobject->object once the create callback has returned success. @@ -257,7 +265,13 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund res_uobj = xa_load(&uctx->cq_xa, res_id); if (res_uobj && res_uobj->object) { cq = container_of(res_uobj->object, struct bnxt_re_cq, ib_cq); - addr = (u64)cq->uctx_cq_page; + if (cq->toggle_entry) + mmap_offset = + rdma_user_mmap_get_offset(&cq->toggle_entry->rdma_entry); + if (mmap_offset) { + kref_get(&cq->toggle_entry->rdma_entry.ref); + toggle_entry = cq->toggle_entry; + } } xa_unlock(&uctx->cq_xa); } else if (res_type == BNXT_RE_SRQ_TOGGLE_MEM) { @@ -267,24 +281,34 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund res_uobj = xa_load(&uctx->srq_xa, res_id); if (res_uobj && res_uobj->object) { srq = container_of(res_uobj->object, struct bnxt_re_srq, ib_srq); - addr = (u64)srq->uctx_srq_page; + if (srq->toggle_entry) + mmap_offset = + rdma_user_mmap_get_offset(&srq->toggle_entry->rdma_entry); + if (mmap_offset) { + kref_get(&srq->toggle_entry->rdma_entry.ref); + toggle_entry = srq->toggle_entry; + } } xa_unlock(&uctx->srq_xa); } else { return -EOPNOTSUPP; } - if (!addr) + if (!mmap_offset) return -EOPNOTSUPP; - entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mem_offset); - if (!entry) + tmem = kzalloc_obj(*tmem); + if (!tmem) { + rdma_user_mmap_entry_put(&toggle_entry->rdma_entry); return -ENOMEM; + } - uobj->object = entry; + tmem->toggle_entry = toggle_entry; + tmem->mmap_offset = mmap_offset; + uobj->object = tmem; uverbs_finalize_uobj_create(attrs, BNXT_RE_TOGGLE_MEM_HANDLE); err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_PAGE, - &mem_offset, sizeof(mem_offset)); + &mmap_offset, sizeof(mmap_offset)); if (err) return err; @@ -305,9 +329,10 @@ static int get_toggle_mem_obj_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { - struct bnxt_re_user_mmap_entry *entry = uobject->object; + struct bnxt_re_toggle_mem *tmem = uobject->object; - rdma_user_mmap_entry_remove(&entry->rdma_entry); + rdma_user_mmap_entry_put(&tmem->toggle_entry->rdma_entry); + kfree(tmem); return 0; }