github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-05 21:15:53 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

RDMA/irdma: Fix inline for multiple SGE's

Browse Source 
[ Upstream commit 4f44e519b6 ]

Currently, inline send and inline write assume a single
SGE and only copy data from the first one. Add support
for multiple SGE's.

Fixes: b48c24c2d7 ("RDMA/irdma: Implement device supported verb APIs")
Signed-off-by: Mustafa Ismail <mustafa.ismail@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
Link: https://lore.kernel.org/r/20221115011701.1379-2-shiraz.saleem@intel.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>
This commit is contained in:
Mustafa Ismail 2022-11-14 19:16:59 -06:00 committed by Greg Kroah-Hartman
parent 3930245bc8
commit 1df9f81555
3 changed files with 119 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
drivers/infiniband/hw/irdma/uk.c
View File

@ -566,21 +566,37 @@ static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe,
/**
* irdma_copy_inline_data_gen_1 - Copy inline data to wqe
* @dest: pointer to wqe
* @src: pointer to inline data
* @len: length of inline data to copy
* @wqe: pointer to wqe
* @sge_list: table of pointers to inline data
* @num_sges: Total inline data length
* @polarity: compatibility parameter
*/
static void irdma_copy_inline_data_gen_1(u8 *dest, u8 *src, u32 len,
u8 polarity)
static void irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list,
u32 num_sges, u8 polarity)
{
if (len <= 16) {
memcpy(dest, src, len);
} else {
memcpy(dest, src, 16);
src += 16;
dest = dest + 32;
memcpy(dest, src, len - 16);
u32 quanta_bytes_remaining = 16;
int i;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
bytes_copied = min(sge_len, quanta_bytes_remaining);
memcpy(wqe, cur_sge, bytes_copied);
wqe += bytes_copied;
cur_sge += bytes_copied;
quanta_bytes_remaining -= bytes_copied;
sge_len -= bytes_copied;
if (!quanta_bytes_remaining) {
/* Remaining inline bytes reside after hdr */
wqe += 16;
quanta_bytes_remaining = 32;
}
}
}
}
@ -612,35 +628,51 @@ static void irdma_set_mw_bind_wqe(__le64 *wqe,
/**
* irdma_copy_inline_data - Copy inline data to wqe
* @dest: pointer to wqe
* @src: pointer to inline data
* @len: length of inline data to copy
* @wqe: pointer to wqe
* @sge_list: table of pointers to inline data
* @num_sges: number of SGE's
* @polarity: polarity of wqe valid bit
*/
static void irdma_copy_inline_data(u8 *dest, u8 *src, u32 len, u8 polarity)
static void irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list,
u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
u32 copy_size;
u32 quanta_bytes_remaining = 8;
bool first_quanta = true;
int i;
dest += 8;
if (len <= 8) {
memcpy(dest, src, len);
return;
}
*((u64 *)dest) = *((u64 *)src);
len -= 8;
src += 8;
dest += 24; /* point to additional 32 byte quanta */
while (len) {
copy_size = len < 31 ? len : 31;
memcpy(dest, src, copy_size);
*(dest + 31) = inline_valid;
len -= copy_size;
dest += 32;
src += copy_size;
wqe += 8;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
bytes_copied = min(sge_len, quanta_bytes_remaining);
memcpy(wqe, cur_sge, bytes_copied);
wqe += bytes_copied;
cur_sge += bytes_copied;
quanta_bytes_remaining -= bytes_copied;
sge_len -= bytes_copied;
if (!quanta_bytes_remaining) {
quanta_bytes_remaining = 31;
/* Remaining inline bytes reside after hdr */
if (first_quanta) {
first_quanta = false;
wqe += 16;
} else {
*wqe = inline_valid;
wqe++;
}
}
}
}
if (!first_quanta && quanta_bytes_remaining < 31)
*(wqe + quanta_bytes_remaining) = inline_valid;
}
/**
@ -679,20 +711,27 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq)
{
__le64 *wqe;
struct irdma_inline_rdma_write *op_info;
struct irdma_rdma_write *op_info;
u64 hdr = 0;
u32 wqe_idx;
bool read_fence = false;
u32 i, total_size = 0;
u16 quanta;
info->push_wqe = qp->push_db ? true : false;
op_info = &info->op.inline_rdma_write;
op_info = &info->op.rdma_write;
if (op_info->len > qp->max_inline_data)
if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
return -EINVAL;
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return -EINVAL;
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
info);
if (!wqe)
return -ENOMEM;
@ -705,7 +744,7 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, op_info->len) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) |
@ -719,7 +758,8 @@ int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
set_64bit_val(wqe, 0,
FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,
qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
op_info->num_lo_sges,
qp->swqe_polarity);
dma_wmb(); /* make sure WQE is populated before valid bit is set */
@ -745,20 +785,27 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq)
{
__le64 *wqe;
struct irdma_post_inline_send *op_info;
struct irdma_post_send *op_info;
u64 hdr;
u32 wqe_idx;
bool read_fence = false;
u32 i, total_size = 0;
u16 quanta;
info->push_wqe = qp->push_db ? true : false;
op_info = &info->op.inline_send;
op_info = &info->op.send;
if (op_info->len > qp->max_inline_data)
if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
return -EINVAL;
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return -EINVAL;
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
info);
if (!wqe)
return -ENOMEM;
@ -773,7 +820,7 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp,
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, op_info->len) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
(info->imm_data_valid ? 1 : 0)) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
@ -789,8 +836,8 @@ int irdma_uk_inline_send(struct irdma_qp_uk *qp,
if (info->imm_data_valid)
set_64bit_val(wqe, 0,
FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len,
qp->swqe_polarity);
qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
op_info->num_sges, qp->swqe_polarity);
dma_wmb(); /* make sure WQE is populated before valid bit is set */

19
drivers/infiniband/hw/irdma/user.h
View File

@ -173,14 +173,6 @@ struct irdma_post_send {
u32 ah_id;
};
struct irdma_post_inline_send {
void *data;
u32 len;
u32 qkey;
u32 dest_qp;
u32 ah_id;
};
struct irdma_post_rq_info {
u64 wr_id;
struct ib_sge *sg_list;
@ -193,12 +185,6 @@ struct irdma_rdma_write {
struct ib_sge rem_addr;
};
struct irdma_inline_rdma_write {
void *data;
u32 len;
struct ib_sge rem_addr;
};
struct irdma_rdma_read {
struct ib_sge *lo_sg_list;
u32 num_lo_sges;
@ -241,8 +227,6 @@ struct irdma_post_sq_info {
struct irdma_rdma_read rdma_read;
struct irdma_bind_window bind_window;
struct irdma_inv_local_stag inv_local_stag;
struct irdma_inline_rdma_write inline_rdma_write;
struct irdma_post_inline_send inline_send;
} op;
};
@ -291,7 +275,8 @@ int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
bool post_sq);
struct irdma_wqe_uk_ops {
void (*iw_copy_inline_data)(u8 *dest, u8 *src, u32 len, u8 polarity);
void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list,
u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
u8 valid);

57
drivers/infiniband/hw/irdma/verbs.c
View File

@ -3136,30 +3136,20 @@ static int irdma_post_send(struct ib_qp *ibqp,
info.stag_to_inv = ib_wr->ex.invalidate_rkey;
}
if (ib_wr->send_flags & IB_SEND_INLINE) {
info.op.inline_send.data = (void *)(unsigned long)
ib_wr->sg_list[0].addr;
info.op.inline_send.len = ib_wr->sg_list[0].length;
if (iwqp->ibqp.qp_type == IB_QPT_UD ||
iwqp->ibqp.qp_type == IB_QPT_GSI) {
ah = to_iwah(ud_wr(ib_wr)->ah);
info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx;
info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey;
info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn;
}
err = irdma_uk_inline_send(ukqp, &info, false);
} else {
info.op.send.num_sges = ib_wr->num_sge;
info.op.send.sg_list = ib_wr->sg_list;
if (iwqp->ibqp.qp_type == IB_QPT_UD ||
iwqp->ibqp.qp_type == IB_QPT_GSI) {
ah = to_iwah(ud_wr(ib_wr)->ah);
info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
}
err = irdma_uk_send(ukqp, &info, false);
info.op.send.num_sges = ib_wr->num_sge;
info.op.send.sg_list = ib_wr->sg_list;
if (iwqp->ibqp.qp_type == IB_QPT_UD ||
iwqp->ibqp.qp_type == IB_QPT_GSI) {
ah = to_iwah(ud_wr(ib_wr)->ah);
info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx;
info.op.send.qkey = ud_wr(ib_wr)->remote_qkey;
info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn;
}
if (ib_wr->send_flags & IB_SEND_INLINE)
err = irdma_uk_inline_send(ukqp, &info, false);
else
err = irdma_uk_send(ukqp, &info, false);
break;
case IB_WR_RDMA_WRITE_WITH_IMM:
if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) {
@ -3176,22 +3166,15 @@ static int irdma_post_send(struct ib_qp *ibqp,
else
info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
if (ib_wr->send_flags & IB_SEND_INLINE) {
info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
info.op.inline_rdma_write.len =
ib_wr->sg_list[0].length;
info.op.inline_rdma_write.rem_addr.addr =
rdma_wr(ib_wr)->remote_addr;
info.op.inline_rdma_write.rem_addr.lkey =
rdma_wr(ib_wr)->rkey;
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
info.op.rdma_write.lo_sg_list = ib_wr->sg_list;
info.op.rdma_write.rem_addr.addr =
rdma_wr(ib_wr)->remote_addr;
info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
if (ib_wr->send_flags & IB_SEND_INLINE)
err = irdma_uk_inline_rdma_write(ukqp, &info, false);
} else {
info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
else
err = irdma_uk_rdma_write(ukqp, &info, false);
}
break;
case IB_WR_RDMA_READ_WITH_INV:
inv_stag = true;