github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

RDMA v7.4 merge window pull request

Browse Source 
A smaller cycle for features, still mostly AI and tool found bug fixes for
 long standing issues:
 
 - Bug fixes and minor cleanups across bnxt_re, cxgb4, EFA, ERDMA,
   HFI1, HNS, ionic, iRDMA, mlx4/mlx5, rdmavt, rtrs, RXE, SIW, SRP,
   isert and IPoIB. Lots of error unwind leaks, integer overflows, and
   use after free during teardown from timers/work that was not
   cancelled
 
 - Core fixes for uverbs DMABUF lifetime, RMPP reassembly use after
   free, multicast rb-tree corruption, restrack, and a user triggerable
   NULL deref when creating a WQ without a CQ
 
 - Prevent userspace from getting a writable mmap of the cxgb4 status
   page and the mlx4 clock page
 
 - Many rxe hardening fixes for bad user and network input, plus
   netdev and tunnel socket lifetime fixes
 
 - Peer supplied values are now validated in rtrs-clt, which is remote
   triggerable
 
 - umem now picks the DMA direction from the access flags instead of
   always using DMA_BIDIRECTIONAL, so an IOMMU can stop the device from
   writing to read-only memory. Several drivers were fixed to pin CQ
   and queue memory as writable to go along with this
 
 - SRQ support, firmware assigned CQ IDs and RCQ userspace support for
   ionic
 
 - RC QP improvements in MANA: MMQ creation, fixed size send WQEs and
   power of two send queues. Also multi-port GSI QPs and a rework of
   the UD send/receive and CQ polling paths
 
 - Relaxed ordering MRs for EFA
 
 - Non-contiguous kernel QP and CQ buffers for ERDMA
 
 - Move the mlx5 data direct infrastructure from mlx5_ib to mlx5_core
   so netdev can use it too. The data direct PCI driver is now named
   "mlx5_dd"
 
 - debugfs knob in hns to fix the SL used by the GSI QP, working around
   a HW problem with dropped CQEs
 
 - Selftest fixes, removal of the qib sysfs ABI docs, DT schema
   conversion for hns, and MANA maintainer handover
 -----BEGIN PGP SIGNATURE-----
 
 iHUEABYKAB0WIQRRRCHOFoQz/8F5bUaFwuHvBreFYQUCatpTYgAKCRCFwuHvBreF
 YYn0AQC5iixdP11KFGUJWf54s/rs4feAM9cHFH/qRN82/enWKwEAmJMuVeVbJurj
 FXds8mDzTFNVez7Vs40GZE9dGbduHAY=
 =m7Zq
 -----END PGP SIGNATURE-----

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull rdma updates from Jason Gunthorpe:
 "A smaller cycle for features, still mostly AI and tool found bug fixes
  for long standing issues:

   - Bug fixes and minor cleanups across bnxt_re, cxgb4, EFA, ERDMA,
     HFI1, HNS, ionic, iRDMA, mlx4/mlx5, rdmavt, rtrs, RXE, SIW, SRP,
     isert and IPoIB. Lots of error unwind leaks, integer overflows, and
     use after free during teardown from timers/work that was not
     cancelled

   - Core fixes for uverbs DMABUF lifetime, RMPP reassembly use after
     free, multicast rb-tree corruption, restrack, and a user
     triggerable NULL deref when creating a WQ without a CQ

   - Prevent userspace from getting a writable mmap of the cxgb4 status
     page and the mlx4 clock page

   - Many rxe hardening fixes for bad user and network input, plus
     netdev and tunnel socket lifetime fixes

   - Peer supplied values are now validated in rtrs-clt, which is remote
     triggerable

   - umem now picks the DMA direction from the access flags instead of
     always using DMA_BIDIRECTIONAL, so an IOMMU can stop the device
     from writing to read-only memory. Several drivers were fixed to pin
     CQ and queue memory as writable to go along with this

   - SRQ support, firmware assigned CQ IDs and RCQ userspace support for
     ionic

   - RC QP improvements in MANA: MMQ creation, fixed size send WQEs and
     power of two send queues. Also multi-port GSI QPs and a rework of
     the UD send/receive and CQ polling paths

   - Relaxed ordering MRs for EFA

   - Non-contiguous kernel QP and CQ buffers for ERDMA

   - Move the mlx5 data direct infrastructure from mlx5_ib to mlx5_core
     so netdev can use it too. The data direct PCI driver is now named
     "mlx5_dd"

   - debugfs knob in hns to fix the SL used by the GSI QP, working
     around a HW problem with dropped CQEs

   - Selftest fixes, removal of the qib sysfs ABI docs, DT schema
     conversion for hns, and MANA maintainer handover"

* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (153 commits)
  RDMA/irdma: Bound in-flight CQP ops across deferred completions
  RDMA/mlx5: Validate priority in STEERING_ANCHOR_CREATE handler
  RDMA/mlx5: Fix CQ resize destination index mask
  RDMA/usnic: Bound buffer size to sizeof(buf) in flowinfo_read()
  RDMA/rxe: Clear pkt->qp before the early drop exits in rxe_rcv()
  dt-bindings: infiniband: Convert hisilicon,hns-roce-v1 to DT schema
  RDMA/srp: Avoid reading uninitialized has_port on srp_parse_in() failure
  RDMA/irdma: Fix PBLE_LEVEL_2 FRWR generation
  RDMA/core: Reject LAG selection when source MAC lookup fails
  RDMA/core: Free unpublished port statistics after initial-read failure
  RDMA/core: Preserve the selected source GID on failed AH modification
  RDMA/uverbs: Fix DMABUF use-after-free during ufile cleanup
  RDMA/uverbs: Fix uobject leak in DMABUF release
  RDMA/irdma: Fix FRWR IRDMAQPSQ_PBLADDR field population
  IB/rdmavt: Fix error unwind path for qp_priv_init() in rvt_create_qp()
  RDMA/irdma: Check MR contiguity before PBLE alloc
  IB/mlx4: Avoid requeueing executing CM timeout work
  RDMA/rxe: bound the WQE opcode before indexing rxe_wr_opcode_info[]
  RDMA/rxe: Fix out-of-range unsigned-to-signed conversion for RDMA message in 2GiB size
  RDMA/rxe: validate inline sge_offset in finish_packet() before indexing inline_data[]
  ...
This commit is contained in:
Linus Torvalds 2026-10-22 13:51:18 -10:00
commit fcb92ecd73
154 changed files with 3829 additions and 2054 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
Documentation/ABI/stable/sysfs-class-infiniband
View File

@ -422,124 +422,6 @@ Description:
information)
=============== =============================================
sysfs interface for Intel IB driver qib
---------------------------------------
What: /sys/class/infiniband/qibX/version
What: /sys/class/infiniband/qibX/hw_rev
What: /sys/class/infiniband/qibX/hca_type
What: /sys/class/infiniband/qibX/board_id
What: /sys/class/infiniband/qibX/boardversion
What: /sys/class/infiniband/qibX/nctxts
What: /sys/class/infiniband/qibX/localbus_info
What: /sys/class/infiniband/qibX/tempsense
What: /sys/class/infiniband/qibX/serial
What: /sys/class/infiniband/qibX/nfreectxts
What: /sys/class/infiniband/qibX/chip_reset
Date: May, 2010
KernelVersion: v2.6.35
Contact: linux-rdma@vger.kernel.org
Description:
=============== ======================================================
version: (RO) Display version information of installed software
and drivers.
hw_rev: (RO) Hardware revision number
hca_type: (RO) Host channel adapter type
board_id: (RO) Manufacturing board id
boardversion: (RO) Current version of the chip architecture
nctxts: (RO) Return the number of user ports (contexts)
available
localbus_info: (RO) Human readable localbus info
tempsense: (RO) Display temp sense registers in decimal
serial: (RO) Serial number of the HCA
nfreectxts: (RO) The number of free user ports (contexts)
available.
chip_reset: (WO) Reset the chip if possible by writing
"reset" to this file. Only allowed if no user
contexts are open that use chip resources.
=============== ======================================================
What: /sys/class/infiniband/qibX/ports/<N>/sl2vl/[0-15]
Date: May, 2010
KernelVersion: v2.6.35
Contact: linux-rdma@vger.kernel.org
Description:
(RO) The directory contains 16 files numbered 0-15 that specify
the Service Level (SL). Listing the SL files returns the Virtual
Lane (VL) as programmed by the SL.
What: /sys/class/infiniband/qibX/ports/<N>/CCMgtA/cc_settings_bin
What: /sys/class/infiniband/qibX/ports/<N>/CCMgtA/cc_table_bin
Date: May, 2010
KernelVersion: v2.6.35
Contact: linux-rdma@vger.kernel.org
Description:
Per-port congestion control. Both are binary attributes.
=============== ================================================
cc_table_bin (RO) Congestion control table size followed by
table entries.
cc_settings_bin (RO) Congestion settings: port control, control
map and an array of 16 entries for the
congestion entries - increase, timer, event log
trigger threshold and the minimum injection rate
delay.
=============== ================================================
What: /sys/class/infiniband/qibX/ports/<N>/linkstate/loopback
What: /sys/class/infiniband/qibX/ports/<N>/linkstate/led_override
What: /sys/class/infiniband/qibX/ports/<N>/linkstate/hrtbt_enable
What: /sys/class/infiniband/qibX/ports/<N>/linkstate/status
What: /sys/class/infiniband/qibX/ports/<N>/linkstate/status_str
Date: May, 2010
KernelVersion: v2.6.35
Contact: linux-rdma@vger.kernel.org
Description:
[to be documented]
=============== ===============================================
loopback: (WO)
led_override: (WO)
hrtbt_enable: (RW)
status: (RO)
status_str: (RO) Displays information about the link state,
possible cable/switch problems, and hardware
errors. Possible states are- "Initted",
"Present", "IB_link_up", "IB_configured" or
"Fatal_Hardware_Error".
=============== ===============================================
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/rc_resends
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/seq_naks
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/rdma_seq
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/rnr_naks
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/other_naks
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/rc_timeouts
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/look_pkts
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/pkt_drops
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/dma_wait
What: /sys/class/infiniband/qibX/ports/<N>/diag_counters/unaligned
Date: May, 2010
KernelVersion: v2.6.35
Contact: linux-rdma@vger.kernel.org
Description:
[to be documented]
sysfs interface for Mellanox Connect-IB HCA driver mlx5
-------------------------------------------------------

173
Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml Normal file
View File

@ -0,0 +1,173 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/infiniband/hisilicon,hns-roce-v1.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: Hisilicon HNS RoCE
maintainers:
- Chengchang Tang <tangchengchang@huawei.com>
- Junxian Huang <huangjunxian6@hisilicon.com>
description:
Hisilicon RoCE engine is a part of network subsystem. It works depending on
other part of network subsystem, such as gmac and dsa fabric.
properties:
compatible:
const: hisilicon,hns-roce-v1
reg:
maxItems: 1
'#address-cells':
const: 2
'#size-cells':
const: 2
dma-coherent: true
interrupts:
items:
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Completion event IRQ
- description: Async event IRQ
- description: Event overflow IRQ
interrupt-names:
items:
- const: hns-roce-comp-0
- const: hns-roce-comp-1
- const: hns-roce-comp-2
- const: hns-roce-comp-3
- const: hns-roce-comp-4
- const: hns-roce-comp-5
- const: hns-roce-comp-6
- const: hns-roce-comp-7
- const: hns-roce-comp-8
- const: hns-roce-comp-9
- const: hns-roce-comp-10
- const: hns-roce-comp-11
- const: hns-roce-comp-12
- const: hns-roce-comp-13
- const: hns-roce-comp-14
- const: hns-roce-comp-15
- const: hns-roce-comp-16
- const: hns-roce-comp-17
- const: hns-roce-comp-18
- const: hns-roce-comp-19
- const: hns-roce-comp-20
- const: hns-roce-comp-21
- const: hns-roce-comp-22
- const: hns-roce-comp-23
- const: hns-roce-comp-24
- const: hns-roce-comp-25
- const: hns-roce-comp-26
- const: hns-roce-comp-27
- const: hns-roce-comp-28
- const: hns-roce-comp-29
- const: hns-roce-comp-30
- const: hns-roce-comp-31
- const: hns-roce-async
- const: hns-roce-common
eth-handle:
description: References to one or more Ethernet devices.
$ref: /schemas/types.yaml#/definitions/phandle-array
items:
maxItems: 1
dsaf-handle:
description: Reference to a dsaf device.
$ref: /schemas/types.yaml#/definitions/phandle
node-guid:
description: A number that uniquely identifies a device or component.
$ref: /schemas/types.yaml#/definitions/uint8-array
required:
- compatible
- reg
- '#address-cells'
- '#size-cells'
- eth-handle
- dsaf-handle
- node-guid
additionalProperties: false
examples:
- |
soc {
#address-cells = <2>;
#size-cells = <2>;
infiniband@c4000000 {
compatible = "hisilicon,hns-roce-v1";
reg = <0x0 0xc4000000 0x0 0x100000>;
#address-cells = <2>;
#size-cells = <2>;
dma-coherent;
eth-handle = <&eth2 &eth3 &eth4 &eth5 &eth6 &eth7>;
dsaf-handle = <&soc0_dsa>;
node-guid = [00 9A CD 00 00 01 02 03];
interrupts = <722 1>, <723 1>, <724 1>, <725 1>,
<726 1>, <727 1>, <728 1>, <729 1>,
<730 1>, <731 1>, <732 1>, <733 1>,
<734 1>, <735 1>, <736 1>, <737 1>,
<738 1>, <739 1>, <740 1>, <741 1>,
<742 1>, <743 1>, <744 1>, <745 1>,
<746 1>, <747 1>, <748 1>, <749 1>,
<750 1>, <751 1>, <752 1>, <753 1>,
<785 1>, <754 4>;
interrupt-names = "hns-roce-comp-0", "hns-roce-comp-1",
"hns-roce-comp-2", "hns-roce-comp-3",
"hns-roce-comp-4", "hns-roce-comp-5",
"hns-roce-comp-6", "hns-roce-comp-7",
"hns-roce-comp-8", "hns-roce-comp-9",
"hns-roce-comp-10", "hns-roce-comp-11",
"hns-roce-comp-12", "hns-roce-comp-13",
"hns-roce-comp-14", "hns-roce-comp-15",
"hns-roce-comp-16", "hns-roce-comp-17",
"hns-roce-comp-18", "hns-roce-comp-19",
"hns-roce-comp-20", "hns-roce-comp-21",
"hns-roce-comp-22", "hns-roce-comp-23",
"hns-roce-comp-24", "hns-roce-comp-25",
"hns-roce-comp-26", "hns-roce-comp-27",
"hns-roce-comp-28", "hns-roce-comp-29",
"hns-roce-comp-30", "hns-roce-comp-31",
"hns-roce-async", "hns-roce-common";
};
};

108
Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt
View File

@ -1,108 +0,0 @@
Hisilicon RoCE DT description
Hisilicon RoCE engine is a part of network subsystem.
It works depending on other part of network subsystem, such as gmac and
dsa fabric.
Additional properties are described here:
Required properties:
- compatible: Should contain "hisilicon,hns-roce-v1".
- reg: Physical base address of the RoCE driver and
length of memory mapped region.
- eth-handle: phandle, specifies a reference to a node
representing a ethernet device.
- dsaf-handle: phandle, specifies a reference to a node
representing a dsaf device.
- node_guid: a number that uniquely identifies a device or component
- #address-cells: must be 2
- #size-cells: must be 2
Optional properties:
- dma-coherent: Present if DMA operations are coherent.
- interrupts: should contain 32 completion event irq,1 async event irq
and 1 event overflow irq.
- interrupt-names:should be one of 34 irqs for roce device
- hns-roce-comp-0 ~ hns-roce-comp-31: 32 complete event irq
- hns-roce-async: 1 async event irq
- hns-roce-common: named common exception warning irq
Example:
infiniband@c4000000 {
compatible = "hisilicon,hns-roce-v1";
reg = <0x0 0xc4000000 0x0 0x100000>;
dma-coherent;
eth-handle = <&eth2 &eth3 &eth4 &eth5 &eth6 &eth7>;
dsaf-handle = <&soc0_dsa>;
node-guid = [00 9A CD 00 00 01 02 03];
#address-cells = <2>;
#size-cells = <2>;
interrupt-parent = <&mbigen_dsa>;
interrupts = <722 1>,
<723 1>,
<724 1>,
<725 1>,
<726 1>,
<727 1>,
<728 1>,
<729 1>,
<730 1>,
<731 1>,
<732 1>,
<733 1>,
<734 1>,
<735 1>,
<736 1>,
<737 1>,
<738 1>,
<739 1>,
<740 1>,
<741 1>,
<742 1>,
<743 1>,
<744 1>,
<745 1>,
<746 1>,
<747 1>,
<748 1>,
<749 1>,
<750 1>,
<751 1>,
<752 1>,
<753 1>,
<785 1>,
<754 4>;
interrupt-names = "hns-roce-comp-0",
"hns-roce-comp-1",
"hns-roce-comp-2",
"hns-roce-comp-3",
"hns-roce-comp-4",
"hns-roce-comp-5",
"hns-roce-comp-6",
"hns-roce-comp-7",
"hns-roce-comp-8",
"hns-roce-comp-9",
"hns-roce-comp-10",
"hns-roce-comp-11",
"hns-roce-comp-12",
"hns-roce-comp-13",
"hns-roce-comp-14",
"hns-roce-comp-15",
"hns-roce-comp-16",
"hns-roce-comp-17",
"hns-roce-comp-18",
"hns-roce-comp-19",
"hns-roce-comp-20",
"hns-roce-comp-21",
"hns-roce-comp-22",
"hns-roce-comp-23",
"hns-roce-comp-24",
"hns-roce-comp-25",
"hns-roce-comp-26",
"hns-roce-comp-27",
"hns-roce-comp-28",
"hns-roce-comp-29",
"hns-roce-comp-30",
"hns-roce-comp-31",
"hns-roce-async",
"hns-roce-common";
};

4
MAINTAINERS
View File

@ -11935,7 +11935,7 @@ M: Chengchang Tang <tangchengchang@huawei.com>
M: Junxian Huang <huangjunxian6@hisilicon.com>
L: linux-rdma@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt
F: Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml
F: drivers/infiniband/hw/hns/
HISILICON SAS Controller
@ -18132,7 +18132,7 @@ F: include/linux/cciss*.h
F: include/uapi/linux/cciss*.h
MICROSOFT MANA RDMA DRIVER
M: Long Li <longli@microsoft.com>
M: Wei Hu <weh@microsoft.com>
M: Konstantin Taranov <kotaranov@microsoft.com>
L: linux-rdma@vger.kernel.org
L: linux-hyperv@vger.kernel.org

34
drivers/infiniband/core/cma.c
View File

@ -4823,6 +4823,29 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,
}
EXPORT_SYMBOL(rdma_reject);
static int cma_force_disconnect(struct rdma_id_private *id_priv)
{
struct cma_work *work;
work = kzalloc_obj(*work);
if (!work)
return -ENOMEM;
INIT_WORK(&work->work, cma_work_handler);
work->old_state = RDMA_CM_CONNECT;
work->new_state = RDMA_CM_DISCONNECT;
work->event.event = RDMA_CM_EVENT_DISCONNECTED;
work->event.status = -ECONNABORTED;
cma_id_get(id_priv);
work->id = id_priv;
trace_cm_force_disconnect(id_priv);
queue_work(cma_wq, &work->work);
return 0;
}
int rdma_disconnect(struct rdma_cm_id *id)
{
struct rdma_id_private *id_priv;
@ -4838,12 +4861,15 @@ int rdma_disconnect(struct rdma_cm_id *id)
goto out;
/* Initiate or respond to a disconnect. */
trace_cm_disconnect(id_priv);
if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
trace_cm_sent_drep(id_priv);
} else {
if (!ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
trace_cm_sent_dreq(id_priv);
goto out;
}
if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) {
trace_cm_sent_drep(id_priv);
goto out;
}
ret = cma_force_disconnect(id_priv);
} else if (rdma_cap_iw_cm(id->device, id->port_num)) {
ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
} else

1
drivers/infiniband/core/cma_trace.h
View File

@ -61,6 +61,7 @@ DEFINE_CMA_FSM_EVENT(send_sidr_rep);
DEFINE_CMA_FSM_EVENT(disconnect);
DEFINE_CMA_FSM_EVENT(sent_drep);
DEFINE_CMA_FSM_EVENT(sent_dreq);
DEFINE_CMA_FSM_EVENT(force_disconnect);
DEFINE_CMA_FSM_EVENT(id_destroy);
TRACE_EVENT(cm_id_attach,

4
drivers/infiniband/core/device.c
View File

@ -893,7 +893,7 @@ static int setup_port_data(struct ib_device *device)
* ib_port_immutable_read() - Read rdma port's immutable data
* @dev: IB device
* @port: port number whose immutable data to read. It starts with index 1 and
* valid upto including rdma_end_port().
* valid up to including rdma_end_port().
*/
const struct ib_port_immutable*
ib_port_immutable_read(struct ib_device *dev, unsigned int port)
@ -3067,7 +3067,7 @@ int ib_add_sub_device(struct ib_device *parent,
sub->parent = parent;
mutex_lock(&parent->subdev_lock);
list_add_tail(&parent->subdev_list_head, &sub->subdev_list);
list_add_tail(&sub->subdev_list, &parent->subdev_list_head);
mutex_unlock(&parent->subdev_lock);
return ret;

22
drivers/infiniband/core/ib_core_uverbs.c
View File

@ -244,16 +244,8 @@ void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry)
entry->driver_removed = true;
xa_unlock(&entry->ucontext->mmap_xa);
list_for_each_entry_safe(uverbs_dmabuf, tmp, &entry->dmabufs, dmabufs_elm) {
dma_resv_lock(uverbs_dmabuf->dmabuf->resv, NULL);
list_del(&uverbs_dmabuf->dmabufs_elm);
uverbs_dmabuf->revoked = true;
dma_buf_invalidate_mappings(uverbs_dmabuf->dmabuf);
dma_resv_wait_timeout(uverbs_dmabuf->dmabuf->resv,
DMA_RESV_USAGE_BOOKKEEP, false,
MAX_SCHEDULE_TIMEOUT);
dma_resv_unlock(uverbs_dmabuf->dmabuf->resv);
kref_put(&uverbs_dmabuf->kref, ib_uverbs_dmabuf_done);
wait_for_completion(&uverbs_dmabuf->comp);
ib_uverbs_dmabuf_revoke(uverbs_dmabuf);
}
mutex_unlock(&entry->dmabufs_lock);
@ -535,18 +527,6 @@ int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs)
}
EXPORT_SYMBOL(uverbs_destroy_def_handler);
/*
* When calling a destroy function during an error unwind we need to pass in
* the udata that is sanitized of all user arguments. Ie from the driver
* perspective it looks like no udata was passed.
*/
struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
{
attrs->driver_udata = (struct ib_udata){};
return &attrs->driver_udata;
}
EXPORT_SYMBOL_NS_GPL(uverbs_get_cleared_udata, "rdma_core");
/**
* _uverbs_alloc() - Quickly allocate memory for use with a bundle
* @bundle: The bundle

2
drivers/infiniband/core/iwcm.c
View File

@ -936,7 +936,7 @@ static void cm_disconnect_handler(struct iwcm_id_private *cm_id_priv,
/*
* CM_ID <-- IDLE
*
* If in the ESTBLISHED or CLOSING states, the QP will have have been
* If in the ESTBLISHED or CLOSING states, the QP will have been
* moved by the provider to the ERR state. Disassociate the CM_ID from
* the QP, move to IDLE, and remove the 'connected' reference.
*

14
drivers/infiniband/core/lag.c
View File

@ -19,6 +19,11 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev,
u8 smac[ETH_ALEN];
bool is_ipv4;
int hdr_len;
int ret;
ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, NULL, smac);
if (ret)
return ERR_PTR(ret);
is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw);
hdr_len = ETH_HLEN + sizeof(struct udphdr) + LL_RESERVED_SPACE(netdev);
@ -26,7 +31,7 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev,
skb = alloc_skb(hdr_len, flags);
if (!skb)
return NULL;
return ERR_PTR(-ENOMEM);
skb->dev = netdev;
skb_reserve(skb, hdr_len);
@ -59,7 +64,7 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev,
ip6h->version = 6;
ip6h->nexthdr = IPPROTO_UDP;
memcpy(&ip6h->flow_lbl, &ah_attr->grh.flow_label,
sizeof(*ip6h->flow_lbl));
sizeof(ip6h->flow_lbl));
memcpy(&ip6h->saddr, ah_attr->grh.sgid_attr->gid.raw,
sizeof(struct in6_addr));
memcpy(&ip6h->daddr, ah_attr->grh.dgid.raw,
@ -70,7 +75,6 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev,
skb_reset_mac_header(skb);
eth = eth_hdr(skb);
skb->protocol = eth->h_proto = htons(is_ipv4 ? ETH_P_IP : ETH_P_IPV6);
rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, NULL, smac);
memcpy(eth->h_source, smac, ETH_ALEN);
memcpy(eth->h_dest, ah_attr->roce.dmac, ETH_ALEN);
@ -86,8 +90,8 @@ static struct net_device *rdma_get_xmit_slave_udp(struct ib_device *device,
struct sk_buff *skb;
skb = rdma_build_skb(master, ah_attr, flags);
if (!skb)
return ERR_PTR(-ENOMEM);
if (IS_ERR(skb))
return ERR_CAST(skb);
rcu_read_lock();
slave = netdev_get_xmit_slave(master, skb,

2
drivers/infiniband/core/mad.c
View File

@ -2342,6 +2342,8 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
recv->header.mapping,
mad_priv_dma_size(recv),
DMA_FROM_DEVICE);
if (unlikely(wc->byte_len < sizeof(struct ib_grh)))
goto out;
/* Setup MAD receive work completion from "normal" work completion */
recv->header.wc = *wc;

7
drivers/infiniband/core/mad_rmpp.c
View File

@ -245,12 +245,15 @@ static void recv_timeout_handler(struct work_struct *work)
struct ib_mad_recv_wc *rmpp_wc;
unsigned long flags;
spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
spin_lock_irqsave(&rmpp_recv->lock, flags);
if (rmpp_recv->state != RMPP_STATE_ACTIVE) {
spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
spin_unlock_irqrestore(&rmpp_recv->lock, flags);
return;
}
rmpp_recv->state = RMPP_STATE_TIMEOUT;
spin_unlock_irqrestore(&rmpp_recv->lock, flags);
spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
list_del(&rmpp_recv->list);
spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);

7
drivers/infiniband/core/multicast.c
View File

@ -189,7 +189,10 @@ static void release_group(struct mcast_group *group)
spin_lock_irqsave(&port->lock, flags);
if (atomic_dec_and_test(&group->refcount)) {
rb_erase(&group->node, &port->table);
if (!RB_EMPTY_NODE(&group->node))
rb_erase(&group->node, &port->table);
spin_unlock_irqrestore(&port->lock, flags);
kfree(group);
deref_port(port);
@ -533,6 +536,8 @@ static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
group->rec = *rec;
if (mgids_changed) {
rb_erase(&group->node, &group->port->table);
RB_CLEAR_NODE(&group->node);
is_mgid0 = !memcmp(&mgid0, &group->rec.mgid,
sizeof(mgid0));
mcast_insert(group->port, group, is_mgid0);

9
drivers/infiniband/core/nldev.c
View File

@ -700,7 +700,8 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin,
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id))
return -EMSGSIZE;
if (!rdma_is_kernel_res(res) &&
if (cq->uobject && cq->uobject->uevent.uobject.context &&
nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
cq->uobject->uevent.uobject.context->res.id))
return -EMSGSIZE;
@ -790,7 +791,7 @@ static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin,
if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id))
goto err;
if (!rdma_is_kernel_res(res) &&
if (pd->uobject && pd->uobject->context &&
nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN,
pd->uobject->context->res.id))
goto err;
@ -1895,8 +1896,10 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
mutex_lock(&nldev_dellink_mutex);
err = device->link_ops->dellink(device);
mutex_unlock(&nldev_dellink_mutex);
if (err)
if (err) {
ib_device_put(device);
return err;
}
}
ib_unregister_device_and_put(device);

2
drivers/infiniband/core/rdma_core.c
View File

@ -734,6 +734,7 @@ void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj
* If this fails then the uobject is still completely valid (though with
* a new ID) and we leak it until context close.
*/
uverbs_get_cleared_udata(attrs);
uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs);
}
EXPORT_SYMBOL_NS_GPL(rdma_assign_uobject, "rdma_core");
@ -750,6 +751,7 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
int ret;
if (hw_obj_valid) {
uverbs_get_cleared_udata(attrs);
ret = uobj->uapi_object->type_class->destroy_hw(
uobj, RDMA_REMOVE_ABORT, attrs);
/*

17
drivers/infiniband/core/rdma_core.h
View File

@ -71,14 +71,19 @@ int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx);
void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile);
#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs);
#else
static inline struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
/*
* When calling a destroy function during an error unwind we need to pass in
* the udata that is sanitized of all user arguments. Ie from the driver
* perspective it looks like no udata was passed.
*/
static inline struct ib_udata *
uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs)
{
return NULL;
if (!attrs)
return NULL;
attrs->driver_udata = (struct ib_udata){};
return &attrs->driver_udata;
}
#endif
/*
* This is the runtime description of the uverbs API, used by the syscall

3
drivers/infiniband/core/restrack.c
View File

@ -250,8 +250,7 @@ void rdma_restrack_add(struct rdma_restrack_entry *res)
ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL);
if (ret)
res->id = 0;
if (qp->qp_type >= IB_QPT_DRIVER)
else if (qp->qp_type >= IB_QPT_DRIVER)
xa_set_mark(&rt->xa, res->id, RESTRACK_DD);
} else if (res->type == RDMA_RESTRACK_COUNTER) {
/* Special case to ensure that cntn points to right counter */

9
drivers/infiniband/core/sysfs.c
View File

@ -996,8 +996,8 @@ static int setup_hw_port_stats(struct ib_port *port,
data->stats->num_counters);
if (ret != data->stats->num_counters) {
if (WARN_ON(ret >= 0))
return -EINVAL;
return ret;
ret = -EINVAL;
goto err_free_data;
}
data->stats->timestamp = jiffies;
@ -1031,6 +1031,11 @@ static int setup_hw_port_stats(struct ib_port *port,
port->hw_stats_data = data;
return 0;
err_free_data:
rdma_free_hw_stats_struct(data->stats);
kfree(data);
return ret;
}
struct rdma_hw_stats *ib_get_hw_stats_port(struct ib_device *ibdev,

77
drivers/infiniband/core/umem.c
View File

@ -55,7 +55,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d
if (dirty)
ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt,
DMA_BIDIRECTIONAL, umem->dma_attrs);
umem->dma_dir, umem->dma_attrs);
for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) {
unpin_user_page_range_dirty_lock(sg_page(sg),
@ -159,9 +159,15 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
}
EXPORT_SYMBOL(ib_umem_find_best_pgsz);
static inline enum dma_data_direction ib_access_dma_dir(int access)
{
return ib_access_writable(access) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
}
static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
unsigned long addr, size_t size,
int access)
int access,
enum dma_data_direction dir)
{
struct ib_umem *umem;
struct page **page_list;
@ -202,6 +208,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
*/
umem->iova = addr;
umem->writable = ib_access_writable(access);
umem->dma_dir = dir;
umem->owning_mm = mm = current->mm;
umem->dma_attrs = DMA_ATTR_REQUIRE_COHERENT;
if (access & IB_ACCESS_RELAXED_ORDERING)
@ -261,7 +268,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
}
ret = ib_dma_map_sgtable_attrs(device, &umem->sgt_append.sgt,
DMA_BIDIRECTIONAL, umem->dma_attrs);
dir, umem->dma_attrs);
if (ret)
goto umem_release;
goto out;
@ -279,17 +286,16 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device,
return ret ? ERR_PTR(ret) : umem;
}
/**
* ib_umem_get_desc - Pin a umem from a buffer descriptor.
* @device: IB device.
* @desc: buffer descriptor (VA or DMABUF).
* @access: IB access flags.
/*
* __ib_umem_get_desc_dir - core implementation for ib_umem_get_desc().
*
* Return: caller-owned umem on success, ERR_PTR(...) on error.
* @dir applies to VA buffers only; dmabuf direction is managed by the
* dmabuf subsystem and this argument is ignored for that type.
*/
struct ib_umem *ib_umem_get_desc(struct ib_device *device,
const struct ib_uverbs_buffer_desc *desc,
int access)
static struct ib_umem *
__ib_umem_get_desc_dir(struct ib_device *device,
const struct ib_uverbs_buffer_desc *desc,
int access, enum dma_data_direction dir)
{
struct ib_umem_dmabuf *umem_dmabuf;
@ -310,11 +316,27 @@ struct ib_umem *ib_umem_get_desc(struct ib_device *device,
return &umem_dmabuf->umem;
case IB_UVERBS_BUFFER_TYPE_VA:
return __ib_umem_get_va(device, desc->addr, desc->length,
access);
access, dir);
default:
return ERR_PTR(-EINVAL);
}
}
/**
* ib_umem_get_desc - Pin a umem from a buffer descriptor.
* @device: IB device.
* @desc: buffer descriptor (VA or DMABUF).
* @access: IB access flags.
*
* Return: caller-owned umem on success, ERR_PTR(...) on error.
*/
struct ib_umem *ib_umem_get_desc(struct ib_device *device,
const struct ib_uverbs_buffer_desc *desc,
int access)
{
return __ib_umem_get_desc_dir(device, desc, access,
ib_access_dma_dir(access));
}
EXPORT_SYMBOL(ib_umem_get_desc);
/*
@ -376,11 +398,12 @@ static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs,
static struct ib_umem *
ib_umem_get_desc_check(struct ib_device *device,
const struct ib_uverbs_buffer_desc *desc,
size_t min_size, int access)
size_t min_size, int access,
enum dma_data_direction dir)
{
struct ib_umem *umem;
umem = ib_umem_get_desc(device, desc, access);
umem = __ib_umem_get_desc_dir(device, desc, access, dir);
if (IS_ERR(umem))
return umem;
if (umem->length < min_size) {
@ -401,7 +424,7 @@ static struct ib_umem *
ib_umem_get_from_attrs(struct ib_device *device,
const struct uverbs_attr_bundle *attrs,
u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler,
size_t size, int access)
size_t size, int access, enum dma_data_direction dir)
{
struct ib_uverbs_buffer_desc desc = {};
int ret;
@ -411,7 +434,7 @@ ib_umem_get_from_attrs(struct ib_device *device,
return NULL;
if (ret)
return ERR_PTR(ret);
return ib_umem_get_desc_check(device, &desc, size, access);
return ib_umem_get_desc_check(device, &desc, size, access, dir);
}
/*
@ -431,7 +454,8 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device,
const struct uverbs_attr_bundle *attrs,
u16 attr_id,
ib_umem_buf_desc_filler_t legacy_filler,
u64 addr, size_t size, int access)
u64 addr, size_t size, int access,
enum dma_data_direction dir)
{
struct ib_uverbs_buffer_desc desc = {};
int ret;
@ -445,7 +469,7 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device,
};
else if (ret)
return ERR_PTR(ret);
return ib_umem_get_desc_check(device, &desc, size, access);
return ib_umem_get_desc_check(device, &desc, size, access, dir);
}
/**
@ -464,7 +488,7 @@ struct ib_umem *ib_umem_get_attr(struct ib_device *device,
u16 attr_id, size_t size, int access)
{
return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size,
access);
access, ib_access_dma_dir(access));
}
EXPORT_SYMBOL(ib_umem_get_attr);
@ -503,7 +527,8 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device,
int access)
{
return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr,
size, access);
size, access,
ib_access_dma_dir(access));
}
EXPORT_SYMBOL(ib_umem_get_attr_or_va);
@ -569,6 +594,9 @@ static int uverbs_create_cq_get_buffer_desc(const struct uverbs_attr_bundle *att
* must arrange its own backing (typically an in-kernel allocation)
* when no source is available.
*
* The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it
* and the CPU only reads.
*
* Return: caller-owned umem on success; NULL when no source supplied
* a buffer; ERR_PTR(...) on error.
*/
@ -579,7 +607,7 @@ struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device,
return ib_umem_get_from_attrs(device, attrs,
UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
uverbs_create_cq_get_buffer_desc,
size, access);
size, access, DMA_FROM_DEVICE);
}
EXPORT_SYMBOL(ib_umem_get_cq_buf);
@ -595,6 +623,9 @@ EXPORT_SYMBOL(ib_umem_get_cq_buf);
* Like ib_umem_get_cq_buf(), but pins @addr/@size when neither the
* UMEM attribute nor the legacy CQ buffer attributes are supplied.
*
* The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it
* and the CPU only reads.
*
* See ib_umem_get_attr_or_va() for the note on @size's dual role and
* the migration path for drivers that would distinguish a user-supplied
* length from a driver-computed minimum.
@ -608,7 +639,7 @@ struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device,
return ib_umem_get_from_attrs_or_va(device, attrs,
UVERBS_ATTR_CREATE_CQ_BUF_UMEM,
uverbs_create_cq_get_buffer_desc,
addr, size, access);
addr, size, access, DMA_FROM_DEVICE);
}
EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va);

37
drivers/infiniband/core/uverbs.h
View File

@ -42,6 +42,8 @@
#include <linux/mutex.h>
#include <linux/completion.h>
#include <linux/cdev.h>
#include <linux/dma-buf.h>
#include <linux/dma-resv.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_umem.h>
@ -135,6 +137,13 @@ struct ib_uverbs_completion_event_file {
struct ib_uverbs_dmabuf_file {
struct ib_uobject uobj;
/*
* Serializes users of dmabuf with the dma_buf's release, which sets it
* to NULL. The uobject can outlive the dma_buf while it waits to be
* destroyed. The map and unmap callbacks use attachment->dmabuf
* instead of taking this lock.
*/
struct mutex dmabuf_release_lock;
struct dma_buf *dmabuf;
struct list_head dmabufs_elm;
struct rdma_user_mmap_entry *mmap_entry;
@ -359,6 +368,34 @@ static inline void ib_uverbs_dmabuf_done(struct kref *kref)
complete(&priv->comp);
}
/*
* Revoke the dmabuf and wait for all of its DMA mappings to be gone. Must be
* called with the mmap entry's dmabufs_lock held, after the dmabuf has been
* removed from the entry's dmabufs list, so it runs once per dmabuf. Lock
* order is dmabufs_lock, dmabuf_release_lock, then the dma_resv lock.
*/
static inline void ib_uverbs_dmabuf_revoke(struct ib_uverbs_dmabuf_file *priv)
{
mutex_lock(&priv->dmabuf_release_lock);
if (priv->dmabuf) {
dma_resv_lock(priv->dmabuf->resv, NULL);
priv->revoked = true;
dma_buf_invalidate_mappings(priv->dmabuf);
dma_resv_wait_timeout(priv->dmabuf->resv,
DMA_RESV_USAGE_BOOKKEEP, false,
MAX_SCHEDULE_TIMEOUT);
dma_resv_unlock(priv->dmabuf->resv);
} else {
/* A released dma_buf has no attachments left to invalidate */
priv->revoked = true;
}
mutex_unlock(&priv->dmabuf_release_lock);
kref_put(&priv->kref, ib_uverbs_dmabuf_done);
/* Wait till all DMA unmaps are completed */
wait_for_completion(&priv->comp);
}
int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason);

1
drivers/infiniband/core/uverbs_std_types_device.c
View File

@ -272,6 +272,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)(
return ret;
ret = ib_init_ucontext(attrs);
if (ret) {
rdma_restrack_put(&attrs->context->res);
kfree(attrs->context);
attrs->context = NULL;
return ret;

46
drivers/infiniband/core/uverbs_std_types_dmabuf.c
View File

@ -28,7 +28,7 @@ uverbs_dmabuf_map(struct dma_buf_attachment *attachment,
struct ib_uverbs_dmabuf_file *priv = attachment->dmabuf->priv;
struct sg_table *ret;
dma_resv_assert_held(priv->dmabuf->resv);
dma_resv_assert_held(attachment->dmabuf->resv);
if (priv->revoked)
return ERR_PTR(-ENODEV);
@ -49,7 +49,7 @@ static void uverbs_dmabuf_unmap(struct dma_buf_attachment *attachment,
{
struct ib_uverbs_dmabuf_file *priv = attachment->dmabuf->priv;
dma_resv_assert_held(priv->dmabuf->resv);
dma_resv_assert_held(attachment->dmabuf->resv);
dma_buf_free_sgt(attachment, sgt, dir);
kref_put(&priv->kref, ib_uverbs_dmabuf_done);
}
@ -68,12 +68,31 @@ static void uverbs_dmabuf_release(struct dma_buf *dmabuf)
struct ib_uverbs_dmabuf_file *priv = dmabuf->priv;
/*
* This can only happen if the fput came from alloc_abort_fd_uobject()
* This can only happen if the fput came from the error unwind of
* UVERBS_METHOD_DMABUF_ALLOC. The abort cleans up the uobject.
*/
if (!priv->uobj.context)
if (!priv)
return;
/*
* uverbs_uobject_release() may drop the last reference to the uobject,
* keep it around until the dmabuf pointer below has been cleared.
*/
uverbs_uobject_get(&priv->uobj);
uverbs_uobject_release(&priv->uobj);
/*
* The dma_buf is freed once this function returns. If the uobject was
* not destroyed above, because the ufile is concurrently being cleaned
* up, it is still reachable through the ufile's uobjects list and the
* mmap entry's dmabufs list. Make sure those paths are not in the
* middle of using the dma_buf and cannot use it afterwards.
*/
mutex_lock(&priv->dmabuf_release_lock);
priv->dmabuf = NULL;
mutex_unlock(&priv->dmabuf_release_lock);
uverbs_uobject_put(&priv->uobj);
}
static const struct dma_buf_ops uverbs_dmabuf_ops = {
@ -99,6 +118,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DMABUF_ALLOC)(
off_t pg_off;
int ret;
mutex_init(&uverbs_dmabuf->dmabuf_release_lock);
ret = uverbs_get_const(&pg_off, attrs, UVERBS_ATTR_ALLOC_DMABUF_PGOFF);
if (ret)
return ret;
@ -141,6 +162,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DMABUF_ALLOC)(
return 0;
err_revoked:
/* Tell uverbs_dmabuf_release() that the uobject is being aborted */
uverbs_dmabuf->dmabuf->priv = NULL;
dma_buf_put(uverbs_dmabuf->dmabuf);
err:
rdma_user_mmap_entry_put(mmap_entry);
@ -162,24 +185,11 @@ static void uverbs_dmabuf_fd_destroy_uobj(struct ib_uobject *uobj,
{
struct ib_uverbs_dmabuf_file *uverbs_dmabuf =
container_of(uobj, struct ib_uverbs_dmabuf_file, uobj);
bool wait_for_comp = false;
mutex_lock(&uverbs_dmabuf->mmap_entry->dmabufs_lock);
dma_resv_lock(uverbs_dmabuf->dmabuf->resv, NULL);
if (!uverbs_dmabuf->revoked) {
uverbs_dmabuf->revoked = true;
list_del(&uverbs_dmabuf->dmabufs_elm);
dma_buf_invalidate_mappings(uverbs_dmabuf->dmabuf);
dma_resv_wait_timeout(uverbs_dmabuf->dmabuf->resv,
DMA_RESV_USAGE_BOOKKEEP, false,
MAX_SCHEDULE_TIMEOUT);
wait_for_comp = true;
}
dma_resv_unlock(uverbs_dmabuf->dmabuf->resv);
if (wait_for_comp) {
kref_put(&uverbs_dmabuf->kref, ib_uverbs_dmabuf_done);
/* Let's wait till all DMA unmap are completed. */
wait_for_completion(&uverbs_dmabuf->comp);
ib_uverbs_dmabuf_revoke(uverbs_dmabuf);
}
mutex_unlock(&uverbs_dmabuf->mmap_entry->dmabufs_lock);

2
drivers/infiniband/core/uverbs_std_types_wq.c
View File

@ -140,7 +140,7 @@ DECLARE_UVERBS_NAMED_METHOD(
UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_CQ_HANDLE,
UVERBS_OBJECT_CQ,
UVERBS_ACCESS_READ,
UA_OPTIONAL),
UA_MANDATORY),
UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_WQ_EVENT_FD,
UVERBS_OBJECT_ASYNC_EVENT,
UVERBS_ACCESS_READ,

2
drivers/infiniband/core/uverbs_uapi.c
View File

@ -729,7 +729,7 @@ void uverbs_disassociate_api(struct uverbs_api *uapi)
*
* release_cleanup is the exception because
* uverbs_uobject_fd_release() needs it. In this case
* the module reference held by the fops will guarentee
* the module reference held by the fops will guarantee
* the type_class remains valid too.
*/
if (type_attrs &&

8
drivers/infiniband/core/verbs.c
View File

@ -1011,7 +1011,8 @@ int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
ah->device->ops.modify_ah(ah, ah_attr) :
-EOPNOTSUPP;
ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr);
if (!ret)
ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr);
rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
return ret;
}
@ -1331,8 +1332,7 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
return qp;
err_security:
qp->device->ops.destroy_qp(
qp, uattrs ? uverbs_get_cleared_udata(uattrs) : NULL);
qp->device->ops.destroy_qp(qp, uverbs_get_cleared_udata(uattrs));
err_create:
rdma_restrack_put(&qp->res);
xa_destroy(&qp->comp_cntrs);
@ -2362,7 +2362,7 @@ EXPORT_SYMBOL(ib_dereg_mr_user);
* @max_num_sg: maximum sg entries available for registration.
*
* Notes:
* Memory registeration page/sg lists must not exceed max_num_sg.
* Memory registration page/sg lists must not exceed max_num_sg.
* For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
* max_num_sg * used_page_size.
*

4
drivers/infiniband/hw/bnxt_re/bnxt_re.h
View File

@ -216,6 +216,10 @@ struct bnxt_re_dev {
unsigned long event_bitmap;
struct bnxt_qplib_cc_param cc_param;
struct workqueue_struct *dcb_wq;
/* Protects dcb_wq against bnxt_re_uninit_dcb_wq() destroying it
* concurrently with bnxt_re_async_notifier() queuing work on it.
*/
spinlock_t dcb_lock;
struct dentry *cc_config;
struct bnxt_re_dbg_cc_config_params *cc_config_params;
struct dentry *cq_coal_cfg;

6
drivers/infiniband/hw/bnxt_re/hw_counters.c
View File

@ -413,7 +413,8 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev,
}
done:
return bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
return bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
rdev->is_virtfn) ?
BNXT_RE_NUM_EXT_COUNTERS : BNXT_RE_NUM_STD_COUNTERS;
}
@ -423,7 +424,8 @@ struct rdma_hw_stats *bnxt_re_ib_alloc_hw_port_stats(struct ib_device *ibdev,
struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
int num_counters = 0;
if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
rdev->is_virtfn))
num_counters = BNXT_RE_NUM_EXT_COUNTERS;
else
num_counters = BNXT_RE_NUM_STD_COUNTERS;

69
drivers/infiniband/hw/bnxt_re/ib_verbs.c
View File

@ -165,10 +165,11 @@ static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev,
qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
}
static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
static u32 bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
struct bnxt_qplib_sge *sg_list, int num)
{
int i, total = 0;
u32 total = 0;
int i;
for (i = 0; i < num; i++) {
sg_list[i].addr = ib_sg_list[i].addr;
@ -2235,6 +2236,14 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
goto exit;
}
if (srq_init_attr->attr.max_sge > dev_attr->max_srq_sges) {
ibdev_err(&rdev->ibdev,
"Create SRQ failed - max_sge %d exceeds supported %d",
srq_init_attr->attr.max_sge, dev_attr->max_srq_sges);
rc = -EINVAL;
goto exit;
}
if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
rc = -EOPNOTSUPP;
goto exit;
@ -2388,6 +2397,7 @@ int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
spin_lock_irqsave(&srq->lock, flags);
while (wr) {
/* Transcribe each ib_recv_wr to qplib_swqe */
wqe.flags = 0;
wqe.num_sge = wr->num_sge;
bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
wqe.wr_id = wr->wr_id;
@ -2903,7 +2913,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
qp->send_psn &= BTH_PSN_MASK;
qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
/* DETH */
/* Use the priviledged Q_Key for QP1 */
/* Use the privileged Q_Key for QP1 */
qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
qp->qp1_hdr.deth.source_qpn = IB_QP1;
@ -3168,8 +3178,9 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
wr->sg_list[i].addr;
sge_len = wr->sg_list[i].length;
if ((sge_len + wqe->inline_len) >
BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
if (sge_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH ||
((sge_len + wqe->inline_len) >
BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH)) {
ibdev_err(&rdev->ibdev,
"Inline data size requested > supported value");
return -EINVAL;
@ -3185,17 +3196,22 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
const struct ib_send_wr *wr,
struct bnxt_qplib_swqe *wqe)
struct bnxt_qplib_swqe *wqe,
u32 *payload_sz)
{
int payload_sz = 0;
int rc;
if (wr->send_flags & IB_SEND_INLINE)
payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
else
payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
wqe->num_sge);
if (wr->send_flags & IB_SEND_INLINE) {
rc = bnxt_re_copy_inline_data(rdev, wr, wqe);
if (rc < 0)
return rc;
*payload_sz = rc;
} else {
*payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
wqe->num_sge);
}
return payload_sz;
return 0;
}
static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
@ -3218,7 +3234,8 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
struct bnxt_re_qp *qp,
const struct ib_send_wr *wr)
{
int rc = 0, payload_sz = 0;
int rc = 0;
u32 payload_sz = 0;
unsigned long flags;
spin_lock_irqsave(&qp->sq_lock, flags);
@ -3234,11 +3251,9 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
goto bad;
}
payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
if (payload_sz < 0) {
rc = -EINVAL;
rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz);
if (rc)
goto bad;
}
wqe.wr_id = wr->wr_id;
wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
@ -3279,7 +3294,8 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
{
struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
struct bnxt_qplib_swqe wqe;
int rc = 0, payload_sz = 0;
int rc = 0;
u32 payload_sz = 0;
unsigned long flags;
spin_lock_irqsave(&qp->sq_lock, flags);
@ -3296,11 +3312,9 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
goto bad;
}
payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
if (payload_sz < 0) {
rc = -EINVAL;
rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz);
if (rc)
goto bad;
}
wqe.wr_id = wr->wr_id;
switch (wr->opcode) {
@ -3749,12 +3763,13 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe,
if (rc)
goto fail;
cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va,
entries * sizeof(struct cq_base),
IB_ACCESS_LOCAL_WRITE);
cq->resize_umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, NULL,
req.cq_va,
entries * sizeof(struct cq_base),
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->resize_umem)) {
rc = PTR_ERR(cq->resize_umem);
ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n",
ibdev_err(&rdev->ibdev, "%s: ib_umem_get_cq_buf_or_va failed! rc = %pe\n",
__func__, cq->resize_umem);
cq->resize_umem = NULL;
goto fail;

26
drivers/infiniband/hw/bnxt_re/main.c
View File

@ -367,9 +367,15 @@ static int bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)
static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev)
{
if (!rdev->dcb_wq)
return;
destroy_workqueue(rdev->dcb_wq);
struct workqueue_struct *dcb_wq;
spin_lock_bh(&rdev->dcb_lock);
dcb_wq = rdev->dcb_wq;
rdev->dcb_wq = NULL;
spin_unlock_bh(&rdev->dcb_lock);
if (dcb_wq)
destroy_workqueue(dcb_wq);
}
static void bnxt_re_dcb_wq_task(struct work_struct *work)
@ -424,14 +430,23 @@ static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *c
switch (event_id) {
case ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE:
dcb_work = kzalloc_obj(*dcb_work, GFP_ATOMIC);
if (!dcb_work)
spin_lock(&rdev->dcb_lock);
if (!rdev->dcb_wq) {
spin_unlock(&rdev->dcb_lock);
break;
}
dcb_work = kzalloc_obj(*dcb_work, GFP_ATOMIC);
if (!dcb_work) {
spin_unlock(&rdev->dcb_lock);
break;
}
dcb_work->rdev = rdev;
memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl));
INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task);
queue_work(rdev->dcb_wq, &dcb_work->work);
spin_unlock(&rdev->dcb_lock);
break;
default:
break;
@ -1448,6 +1463,7 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct auxiliary_device *adev,
INIT_LIST_HEAD(&rdev->qp_list);
mutex_init(&rdev->qp_lock);
mutex_init(&rdev->pacing.dbq_lock);
spin_lock_init(&rdev->dcb_lock);
atomic_set(&rdev->stats.res.qp_count, 0);
atomic_set(&rdev->stats.res.cq_count, 0);
atomic_set(&rdev->stats.res.srq_count, 0);

4
drivers/infiniband/hw/bnxt_re/qplib_rcfw.c
View File

@ -638,6 +638,10 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
err_event = (struct creq_qp_error_notification *)qp_event;
qp_id = le32_to_cpu(err_event->xid);
spin_lock(&rcfw->tbl_lock);
if (!rcfw->qp_tbl) {
spin_unlock(&rcfw->tbl_lock);
break;
}
tbl_indx = map_qp_id_to_tbl_indx(qp_id, rcfw);
qp = rcfw->qp_tbl[tbl_indx].qp_handle;
if (!qp) {

23
drivers/infiniband/hw/bnxt_re/qplib_res.c
View File

@ -45,6 +45,7 @@
#include <linux/dma-mapping.h>
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>
#include <linux/bitops.h>
#include <rdma/ib_verbs.h>
#include <rdma/iter.h>
@ -668,9 +669,9 @@ static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res,
{
u32 bytes;
bytes = max >> 3;
bytes = BITS_TO_LONGS(max) * sizeof(unsigned long);
if (!bytes)
bytes = 1;
bytes = sizeof(unsigned long);
pdt->tbl = kmalloc(bytes, GFP_KERNEL);
if (!pdt->tbl)
return -ENOMEM;
@ -848,9 +849,9 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
if (!dpit->app_tbl)
return -ENOMEM;
bytes = dpit->max >> 3;
bytes = BITS_TO_LONGS(dpit->max) * sizeof(unsigned long);
if (!bytes)
bytes = 1;
bytes = sizeof(unsigned long);
dpit->tbl = kmalloc(bytes, GFP_KERNEL);
if (!dpit->tbl) {
@ -909,7 +910,15 @@ int bnxt_qplib_init_res(struct bnxt_qplib_res *res)
void bnxt_qplib_free_res(struct bnxt_qplib_res *res)
{
kfree(res->rcfw->qp_tbl);
struct bnxt_qplib_rcfw *rcfw = res->rcfw;
struct bnxt_qplib_qp_node *qp_tbl;
/* The CREQ tasklet may still be running; unpublish the table first */
spin_lock_bh(&rcfw->tbl_lock);
qp_tbl = rcfw->qp_tbl;
rcfw->qp_tbl = NULL;
spin_unlock_bh(&rcfw->tbl_lock);
kvfree(qp_tbl);
bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl);
bnxt_qplib_free_pd_tbl(&res->pd_tbl);
bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl);
@ -926,8 +935,8 @@ int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct net_device *netdev)
/* Allocate one extra to hold the QP1 entries */
rcfw->qp_tbl_size = max_t(u32, BNXT_RE_MAX_QPC_COUNT + 1, dev_attr->max_qp);
rcfw->qp_tbl = kzalloc_objs(struct bnxt_qplib_qp_node,
rcfw->qp_tbl_size);
rcfw->qp_tbl = kvzalloc_objs(struct bnxt_qplib_qp_node,
rcfw->qp_tbl_size);
if (!rcfw->qp_tbl)
return -ENOMEM;

2
drivers/infiniband/hw/bnxt_re/qplib_sp.c
View File

@ -160,7 +160,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
attr->max_srq = le16_to_cpu(sb->max_srq);
attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
attr->max_srq_sges = sb->max_srq_sge;
attr->max_srq_sges = min_t(u32, sb->max_srq_sge, BNXT_STATIC_MAX_SGE);
attr->max_pkey = 1;
attr->max_inline_data = attr->max_qp_sges * sizeof(struct sq_sge);
if (!bnxt_qplib_is_chip_gen_p7(rcfw->res->cctx))

5
drivers/infiniband/hw/cxgb4/cm.c
View File

@ -4006,8 +4006,10 @@ static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb,
int ret;
req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL);
if (!req_skb)
if (!req_skb) {
kfree_skb(skb);
return;
}
req = __skb_put_zero(req_skb, sizeof(*req));
req->op_compl = htonl(WR_OP_V(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL_F);
req->len16_pkd = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)));
@ -4156,6 +4158,7 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb)
pdev = ip_dev_find(&init_net, iph->daddr);
if (!pdev) {
pr_err("%s - failed to find device!\n", __func__);
neigh_release(neigh);
goto free_dst;
}
e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh,

2
drivers/infiniband/hw/cxgb4/cq.c
View File

@ -1182,7 +1182,7 @@ void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx)
struct c4iw_cq *rchp = to_c4iw_cq(qhp->ibqp.recv_cq);
unsigned long flag;
/* locking heirarchy: cq lock first, then qp lock. */
/* locking hierarchy: cq lock first, then qp lock. */
spin_lock_irqsave(&rchp->lock, flag);
spin_lock(&qhp->lock);

5
drivers/infiniband/hw/cxgb4/provider.c
View File

@ -171,6 +171,11 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
len, t4_pgprot_wc(vma->vm_page_prot));
break;
case CXGB4_MMAP_CONTIG:
if (vma->vm_flags & (VM_WRITE | VM_EXEC)) {
ret = -EPERM;
break;
}
vm_flags_clear(vma, VM_MAYWRITE);
ret = io_remap_pfn_range(vma, vma->vm_start,
addr >> PAGE_SHIFT,
len, vma->vm_page_prot);

5
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
View File

@ -358,7 +358,8 @@ struct efa_admin_reg_mr_cmd {
* 5:0 : phys_page_size_shift - page size is (1 <<
* phys_page_size_shift). Page size is used for
* building the Virtual to Physical address mapping
* 6 : reserved - MBZ
* 6 : relaxed_ordering_enable - Enable PCIe relaxed
* ordering for this memory region
* 7 : mem_addr_phy_mode_en - Enable bit for physical
* memory registration (no translation), can be used
* only by privileged clients. If set, PBL must
@ -1203,6 +1204,7 @@ struct efa_admin_modify_event_counter_resp {
/* reg_mr_cmd */
#define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(5, 0)
#define EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE_MASK BIT(6)
#define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK BIT(7)
#define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK BIT(0)
#define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK BIT(1)
@ -1231,7 +1233,6 @@ struct efa_admin_modify_event_counter_resp {
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK BIT(3)
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4)
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_EVENT_COUNTERS_MASK BIT(5)
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_SHIFT 10
#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_MASK BIT(10)
/* feature_queue_attr_desc_2 */

28
drivers/infiniband/hw/efa/efa_com.c
View File

@ -41,6 +41,7 @@ enum efa_cmd_status {
EFA_CMD_ALLOCATED,
EFA_CMD_SUBMITTED,
EFA_CMD_COMPLETED,
EFA_CMD_TIMED_OUT,
};
struct efa_comp_ctx {
@ -591,6 +592,7 @@ static int efa_com_comp_status_to_errno(u8 comp_status)
static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx,
struct efa_com_admin_queue *aq)
{
bool comp_timeout = false;
unsigned long timeout;
unsigned long flags;
@ -599,12 +601,19 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c
while (1) {
spin_lock_irqsave(&aq->cq.lock, flags);
efa_com_handle_admin_completion(aq);
spin_unlock_irqrestore(&aq->cq.lock, flags);
if (comp_ctx->status != EFA_CMD_SUBMITTED)
if (comp_ctx->status != EFA_CMD_SUBMITTED) {
spin_unlock_irqrestore(&aq->cq.lock, flags);
break;
}
if (time_is_before_jiffies(timeout)) {
comp_ctx->status = EFA_CMD_TIMED_OUT;
comp_timeout = true;
}
spin_unlock_irqrestore(&aq->cq.lock, flags);
if (comp_timeout) {
ibdev_err_ratelimited(
aq->efa_dev,
"Wait for completion (polling) timeout\n");
@ -625,10 +634,11 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c
static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx,
struct efa_com_admin_queue *aq)
{
unsigned long flags;
unsigned long flags, remaining_jiffies;
bool completed;
wait_for_completion_timeout(&comp_ctx->wait_event,
usecs_to_jiffies(aq->completion_timeout));
remaining_jiffies = wait_for_completion_timeout(
&comp_ctx->wait_event, usecs_to_jiffies(aq->completion_timeout));
/*
* In case the command wasn't completed find out the root cause.
@ -636,14 +646,16 @@ static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *com
* 1) No completion (timeout reached)
* 2) There is completion but the device didn't get any msi-x interrupt.
*/
if (comp_ctx->status == EFA_CMD_SUBMITTED) {
if (remaining_jiffies == 0) {
spin_lock_irqsave(&aq->cq.lock, flags);
efa_com_handle_admin_completion(aq);
completed = (comp_ctx->status == EFA_CMD_COMPLETED);
comp_ctx->status = EFA_CMD_TIMED_OUT;
spin_unlock_irqrestore(&aq->cq.lock, flags);
atomic64_inc(&aq->stats.no_completion);
if (comp_ctx->status == EFA_CMD_COMPLETED)
if (completed)
ibdev_err_ratelimited(
aq->efa_dev,
"The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",

9
drivers/infiniband/hw/efa/efa_com_cmd.c
View File

@ -236,8 +236,15 @@ int efa_com_register_mr(struct efa_com_dev *edev,
mr_cmd.mr_length = params->mr_length_in_bytes;
EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT,
params->page_shift);
if (params->relaxed_ordering)
EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE, 1);
mr_cmd.iova = params->iova;
mr_cmd.permissions = params->permissions;
if (params->permissions.local_write)
EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE, 1);
if (params->permissions.remote_write)
EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE, 1);
if (params->permissions.remote_read)
EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE, 1);
if (params->inline_pbl) {
memcpy(mr_cmd.pbl.inline_pbl_array,

8
drivers/infiniband/hw/efa/efa_com_cmd.h
View File

@ -204,8 +204,12 @@ struct efa_com_reg_mr_params {
* address mapping
*/
u8 page_shift;
/* see permissions field of struct efa_admin_reg_mr_cmd */
u8 permissions;
struct {
u8 local_write : 1;
u8 remote_write : 1;
u8 remote_read : 1;
} permissions;
u8 relaxed_ordering;
u8 inline_pbl;
u8 indirect;
};

7
drivers/infiniband/hw/efa/efa_verbs.c
View File

@ -190,7 +190,7 @@ static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr,
{
void *addr;
addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL);
if (!addr)
return NULL;
@ -1676,7 +1676,10 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,
params.pd = to_epd(ibpd)->pdn;
params.iova = virt_addr;
params.mr_length_in_bytes = length;
params.permissions = access_flags;
params.permissions.local_write = !!(access_flags & IB_ACCESS_LOCAL_WRITE);
params.permissions.remote_write = !!(access_flags & IB_ACCESS_REMOTE_WRITE);
params.permissions.remote_read = !!(access_flags & IB_ACCESS_REMOTE_READ);
params.relaxed_ordering = !!(access_flags & IB_ACCESS_RELAXED_ORDERING);
pg_sz = ib_umem_find_best_pgsz(mr->umem,
dev->dev_attr.page_size_cap,

18
drivers/infiniband/hw/erdma/erdma_cq.c
View File

@ -8,8 +8,8 @@
static void *get_next_valid_cqe(struct erdma_cq *cq)
{
__be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,
cq->depth, CQE_SHIFT);
__be32 *cqe = erdma_kmem_get_entry(
&cq->kern_cq.qbuf_mem, cq->kern_cq.ci, cq->depth, CQE_SHIFT);
u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
be32_to_cpu(READ_ONCE(*cqe)));
@ -161,8 +161,8 @@ static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc)
if (qtype == ERDMA_CQE_QTYPE_SQ) {
id_table = kern_qp->swr_tbl;
depth = qp->attrs.sq_size;
wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
qp->attrs.sq_size, SQEBB_SHIFT);
wqe_hdr = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
qp->attrs.sq_size, SQEBB_SHIFT);
kern_qp->sq_ci =
FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) +
wqe_idx + 1;
@ -242,15 +242,15 @@ void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn)
while (ncqe > 0) {
cur_cq_ci = prev_cq_ci + ncqe - 1;
cqe = get_queue_entry(cq->kern_cq.qbuf, cur_cq_ci, cq->depth,
CQE_SHIFT);
cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem, cur_cq_ci,
cq->depth, CQE_SHIFT);
if (be32_to_cpu(cqe->qpn) == qpn) {
++nqp_cqe;
} else if (nqp_cqe) {
dst_cqe = get_queue_entry(cq->kern_cq.qbuf,
cur_cq_ci + nqp_cqe,
cq->depth, CQE_SHIFT);
dst_cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem,
cur_cq_ci + nqp_cqe,
cq->depth, CQE_SHIFT);
owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
be32_to_cpu(dst_cqe->hdr));
cqe->hdr = cpu_to_be32(

2
drivers/infiniband/hw/erdma/erdma_main.c
View File

@ -50,7 +50,7 @@ static int erdma_enum_and_get_netdev(struct erdma_dev *dev)
struct net_device *netdev;
int ret = -EPROBE_DEFER;
/* Already binded to a net_device, so we skip. */
/* Already bound to a net_device, so we skip. */
if (dev->netdev)
return 0;

38
drivers/infiniband/hw/erdma/erdma_qp.c
View File

@ -259,8 +259,8 @@ static void erdma_reset_qp(struct erdma_qp *qp)
qp->kern_qp.rq_ci = 0;
memset(qp->kern_qp.swr_tbl, 0, qp->attrs.sq_size * sizeof(u64));
memset(qp->kern_qp.rwr_tbl, 0, qp->attrs.rq_size * sizeof(u64));
memset(qp->kern_qp.sq_buf, 0, qp->attrs.sq_size << SQEBB_SHIFT);
memset(qp->kern_qp.rq_buf, 0, qp->attrs.rq_size << RQE_SHIFT);
erdma_kmem_clear(&qp->kern_qp.sq_mem);
erdma_kmem_clear(&qp->kern_qp.rq_mem);
erdma_remove_cqes_of_qp(&qp->scq->ibcq, QP_ID(qp));
if (qp->rcq != qp->scq)
erdma_remove_cqes_of_qp(&qp->rcq->ibcq, QP_ID(qp));
@ -332,8 +332,8 @@ static int fill_inline_data(struct erdma_qp *qp,
wqe_idx += (sgl_offset >> SQEBB_SHIFT);
sgl_offset &= (SQEBB_SIZE - 1);
data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size,
SQEBB_SHIFT);
data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
qp->attrs.sq_size, SQEBB_SHIFT);
while (i < send_wr->num_sge) {
bytes += send_wr->sg_list[i].length;
@ -356,8 +356,9 @@ static int fill_inline_data(struct erdma_qp *qp,
wqe_idx += (sgl_offset >> SQEBB_SHIFT);
sgl_offset &= (SQEBB_SIZE - 1);
data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
qp->attrs.sq_size, SQEBB_SHIFT);
data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem,
wqe_idx, qp->attrs.sq_size,
SQEBB_SHIFT);
if (!remain_size)
break;
}
@ -385,8 +386,8 @@ static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr,
while (i < send_wr->num_sge) {
wqe_idx += (sgl_offset >> SQEBB_SHIFT);
sgl_offset &= (SQEBB_SIZE - 1);
sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
qp->attrs.sq_size, SQEBB_SHIFT);
sgl = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx,
qp->attrs.sq_size, SQEBB_SHIFT);
bytes += send_wr->sg_list[i].length;
memcpy(sgl + sgl_offset, &send_wr->sg_list[i],
@ -463,8 +464,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
send_wr->opcode != IB_WR_SEND_WITH_IMM)
return -EINVAL;
entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size,
SQEBB_SHIFT);
entry = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx,
qp->attrs.sq_size, SQEBB_SHIFT);
/* Clear the SQE header section. */
*entry = 0;
@ -524,8 +525,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
read_sqe->sink_to_h =
cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr));
sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
qp->attrs.sq_size, SQEBB_SHIFT);
sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1,
qp->attrs.sq_size, SQEBB_SHIFT);
sge->addr = cpu_to_le64(rdma_wr->remote_addr);
sge->key = cpu_to_le32(rdma_wr->rkey);
sge->length = cpu_to_le32(send_wr->sg_list[0].length);
@ -569,8 +570,9 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) {
attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);
/* Copy SGLs to SQE content to accelerate */
memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
qp->attrs.sq_size, SQEBB_SHIFT),
memcpy(erdma_kmem_get_entry(&qp->kern_qp.sq_mem,
idx + 1, qp->attrs.sq_size,
SQEBB_SHIFT),
mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents));
wqe_size = sizeof(struct erdma_reg_mr_sqe) +
MTT_SIZE(mr->mem.mtt_nents);
@ -605,8 +607,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
cpu_to_le64(atomic_wr(send_wr)->compare_add);
}
sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
qp->attrs.sq_size, SQEBB_SHIFT);
sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1,
qp->attrs.sq_size, SQEBB_SHIFT);
sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr);
sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey);
sge++;
@ -702,8 +704,8 @@ static int erdma_post_recv_one(struct erdma_qp *qp,
const struct ib_recv_wr *recv_wr)
{
struct erdma_rqe *rqe =
get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi,
qp->attrs.rq_size, RQE_SHIFT);
erdma_kmem_get_entry(&qp->kern_qp.rq_mem, qp->kern_qp.rq_pi,
qp->attrs.rq_size, RQE_SHIFT);
rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1);
rqe->qpn = cpu_to_le32(QP_ID(qp));

583
drivers/infiniband/hw/erdma/erdma_verbs.c
View File

@ -41,7 +41,7 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
struct erdma_dev *dev = to_edev(qp->ibqp.device);
struct erdma_pd *pd = to_epd(qp->ibqp.pd);
struct erdma_cmdq_create_qp_req req;
struct erdma_uqp *user_qp;
struct erdma_mem *sq_mem, *rq_mem;
u64 resp0, resp1;
int err;
@ -63,67 +63,46 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
ERDMA_QPT_UD);
if (rdma_is_kernel_res(&qp->ibqp.res)) {
u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
req.sq_cqn_mtt_cfg =
FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
pgsz_range) |
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
req.rq_cqn_mtt_cfg =
FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
pgsz_range) |
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
req.sq_mtt_cfg =
FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
ERDMA_MR_MTT_0LEVEL);
req.rq_mtt_cfg = req.sq_mtt_cfg;
req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
sq_mem = &qp->kern_qp.sq_mem;
rq_mem = &qp->kern_qp.rq_mem;
req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma;
req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma;
} else {
user_qp = &qp->user_qp;
req.sq_cqn_mtt_cfg = FIELD_PREP(
ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
sq_mem = &qp->user_qp.sq_mem;
rq_mem = &qp->user_qp.rq_mem;
req.sq_dbrec_dma = qp->user_qp.sq_dbrec_dma;
req.rq_dbrec_dma = qp->user_qp.rq_dbrec_dma;
}
req.sq_cqn_mtt_cfg =
FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
ilog2(sq_mem->page_size) - ERDMA_HW_PAGE_SHIFT);
req.sq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
req.rq_cqn_mtt_cfg =
FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
ilog2(rq_mem->page_size) - ERDMA_HW_PAGE_SHIFT);
req.rq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
req.sq_mtt_cfg =
sq_mem->page_offset |
FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, sq_mem->mtt_nents);
req.rq_mtt_cfg =
rq_mem->page_offset |
FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, rq_mem->mtt_nents);
assemble_qbuf_mtt_for_cmd(sq_mem, &req.sq_mtt_cfg, &req.sq_buf_addr,
req.sq_mtt_entry);
assemble_qbuf_mtt_for_cmd(rq_mem, &req.rq_mtt_cfg, &req.rq_buf_addr,
req.rq_mtt_entry);
if (uctx && uctx->ext_db.enable) {
req.sq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
req.rq_cqn_mtt_cfg = FIELD_PREP(
ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
req.rq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
user_qp->sq_mem.mtt_nents);
req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
user_qp->rq_mem.mtt_nents);
assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
&req.sq_buf_addr, req.sq_mtt_entry);
assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
&req.rq_buf_addr, req.rq_mtt_entry);
req.sq_dbrec_dma = user_qp->sq_dbrec_dma;
req.rq_dbrec_dma = user_qp->rq_dbrec_dma;
if (uctx->ext_db.enable) {
req.sq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
req.db_cfg =
FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
uctx->ext_db.sdb_off) |
FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
uctx->ext_db.rdb_off);
}
FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
req.db_cfg = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
uctx->ext_db.sdb_off) |
FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
uctx->ext_db.rdb_off);
}
err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1,
@ -194,7 +173,6 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
struct erdma_dev *dev = to_edev(cq->ibcq.device);
struct erdma_cmdq_create_cq_req req;
struct erdma_mem *mem;
u32 page_size;
erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
CMDQ_OPCODE_CREATE_CQ);
@ -204,48 +182,34 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
if (rdma_is_kernel_res(&cq->ibcq.res)) {
page_size = SZ_32M;
req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
ERDMA_MR_MTT_0LEVEL);
req.first_page_offset = 0;
mem = &cq->kern_cq.qbuf_mem;
req.cq_dbrec_dma = cq->kern_cq.dbrec_dma;
} else {
mem = &cq->user_cq.qbuf_mem;
req.cfg0 |=
FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
if (mem->mtt_nents == 1) {
req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
req.cfg1 |=
FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
ERDMA_MR_MTT_0LEVEL);
} else {
req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
req.cfg1 |=
FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
ERDMA_MR_MTT_1LEVEL);
}
req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
mem->mtt_nents);
req.first_page_offset = mem->page_offset;
req.cq_dbrec_dma = cq->user_cq.dbrec_dma;
}
if (uctx->ext_db.enable) {
req.cfg1 |= FIELD_PREP(
ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
uctx->ext_db.cdb_off);
}
req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
if (mem->mtt_nents == 1) {
req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
ERDMA_MR_MTT_0LEVEL);
} else {
req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
ERDMA_MR_MTT_1LEVEL);
}
req.cfg1 |=
FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, mem->mtt_nents);
req.first_page_offset = mem->page_offset;
if (uctx && uctx->ext_db.enable) {
req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
uctx->ext_db.cdb_off);
}
return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
@ -505,96 +469,28 @@ static int erdma_qp_validate_attr(struct erdma_dev *dev,
return 0;
}
static void free_kernel_qp(struct erdma_qp *qp)
static void erdma_init_mtt_leaf(struct erdma_mem *mem, struct erdma_mtt *mtt)
{
struct erdma_dev *dev = qp->dev;
vfree(qp->kern_qp.swr_tbl);
vfree(qp->kern_qp.rwr_tbl);
if (qp->kern_qp.sq_buf)
dma_free_coherent(&dev->pdev->dev,
qp->attrs.sq_size << SQEBB_SHIFT,
qp->kern_qp.sq_buf,
qp->kern_qp.sq_buf_dma_addr);
if (qp->kern_qp.sq_dbrec)
dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
qp->kern_qp.sq_dbrec_dma);
if (qp->kern_qp.rq_buf)
dma_free_coherent(&dev->pdev->dev,
qp->attrs.rq_size << RQE_SHIFT,
qp->kern_qp.rq_buf,
qp->kern_qp.rq_buf_dma_addr);
if (qp->kern_qp.rq_dbrec)
dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
qp->kern_qp.rq_dbrec_dma);
}
static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
struct ib_qp_init_attr *attrs)
{
struct erdma_kqp *kqp = &qp->kern_qp;
int size;
if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
kqp->sig_all = 1;
kqp->sq_pi = 0;
kqp->sq_ci = 0;
kqp->rq_pi = 0;
kqp->rq_ci = 0;
kqp->hw_sq_db =
dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
if (!kqp->swr_tbl || !kqp->rwr_tbl)
goto err_out;
size = qp->attrs.sq_size << SQEBB_SHIFT;
kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
&kqp->sq_buf_dma_addr, GFP_KERNEL);
if (!kqp->sq_buf)
goto err_out;
kqp->sq_dbrec =
dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
if (!kqp->sq_dbrec)
goto err_out;
size = qp->attrs.rq_size << RQE_SHIFT;
kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
&kqp->rq_buf_dma_addr, GFP_KERNEL);
if (!kqp->rq_buf)
goto err_out;
kqp->rq_dbrec =
dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
if (!kqp->rq_dbrec)
goto err_out;
return 0;
err_out:
free_kernel_qp(qp);
return -ENOMEM;
}
static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
{
struct erdma_mtt *mtt = mem->mtt;
struct ib_block_iter biter;
u32 idx = 0;
if (mem->type == ERDMA_UMEM) {
rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
} else {
for (; idx < mem->page_cnt; idx++)
mtt->buf[idx] = mem->kmem.buf_list[idx].dma_addr;
}
}
static void erdma_init_bottom_mtt(struct erdma_mem *mem)
{
struct erdma_mtt *mtt = mem->mtt;
while (mtt->low_level)
mtt = mtt->low_level;
rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
erdma_init_mtt_leaf(mem, mtt);
}
static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
@ -826,54 +722,216 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
}
}
static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
u64 start, u64 len, int access, u64 virt,
unsigned long req_page_size, bool force_continuous)
static void erdma_kmem_free(struct erdma_dev *dev, struct erdma_mem *mem)
{
int ret = 0;
struct erdma_buf_list *buf_list = mem->kmem.buf_list;
u32 i;
mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access);
if (IS_ERR(mem->umem)) {
ret = PTR_ERR(mem->umem);
mem->umem = NULL;
return ret;
for (i = 0; i < mem->page_cnt; i++)
dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf,
buf_list[i].dma_addr);
kfree(buf_list);
}
static int erdma_kmem_alloc(struct erdma_dev *dev, struct erdma_mem *mem,
u32 page_cnt)
{
struct erdma_buf_list *buf_list;
u32 i;
buf_list = kcalloc(page_cnt, sizeof(*buf_list), GFP_KERNEL);
if (!buf_list)
return -ENOMEM;
for (i = 0; i < page_cnt; i++) {
buf_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
&buf_list[i].dma_addr,
GFP_KERNEL);
if (!buf_list[i].buf)
goto err_free_pages;
}
mem->va = virt;
mem->len = len;
mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
mem->page_offset = start & (mem->page_size - 1);
mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
mem->kmem.buf_list = buf_list;
return 0;
err_free_pages:
while (i--)
dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf,
buf_list[i].dma_addr);
kfree(buf_list);
return -ENOMEM;
}
static void erdma_mem_free(struct erdma_dev *dev, struct erdma_mem *mem)
{
switch (mem->type) {
case ERDMA_UMEM:
ib_umem_release(mem->umem);
break;
case ERDMA_KMEM:
erdma_kmem_free(dev, mem);
break;
default:
break;
}
}
static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
struct erdma_mem_init_attr *attr)
{
struct erdma_mtt *mtt;
int ret;
mem->type = attr->type;
switch (mem->type) {
case ERDMA_UMEM:
if (attr->flags & ERDMA_MEM_FLAG_CQ_BUF)
mem->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
attr->start,
attr->len,
attr->access);
else
mem->umem = ib_umem_get_va(&dev->ibdev, attr->start,
attr->len, attr->access);
if (IS_ERR(mem->umem)) {
ret = PTR_ERR(mem->umem);
return ret;
}
mem->page_size = ib_umem_find_best_pgsz(
mem->umem, attr->req_page_size, attr->virt);
mem->mtt_nents =
ib_umem_num_dma_blocks(mem->umem, mem->page_size);
break;
case ERDMA_KMEM:
mem->page_size = PAGE_SIZE;
mem->mtt_nents = DIV_ROUND_UP(attr->len, PAGE_SIZE);
ret = erdma_kmem_alloc(dev, mem, mem->mtt_nents);
if (ret)
return ret;
break;
default:
return -EINVAL;
}
mem->va = attr->virt;
mem->len = attr->len;
mem->page_offset = attr->start & (mem->page_size - 1);
mem->page_cnt = mem->mtt_nents;
mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
force_continuous);
if (IS_ERR(mem->mtt)) {
ret = PTR_ERR(mem->mtt);
goto error_ret;
mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
!(attr->flags & ERDMA_MEM_FLAG_MR_BUF));
if (IS_ERR(mtt)) {
ret = PTR_ERR(mtt);
goto err_free_mem;
}
mem->mtt = mtt;
erdma_fill_bottom_mtt(dev, mem);
erdma_init_bottom_mtt(mem);
if (mtt->continuous)
dma_sync_single_for_device(&dev->pdev->dev, mtt->buf_dma,
mtt->size, DMA_TO_DEVICE);
return 0;
error_ret:
if (mem->umem) {
ib_umem_release(mem->umem);
mem->umem = NULL;
}
err_free_mem:
erdma_mem_free(dev, mem);
return ret;
}
static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem)
{
if (mem->mtt)
erdma_destroy_mtt(dev, mem->mtt);
erdma_destroy_mtt(dev, mem->mtt);
erdma_mem_free(dev, mem);
}
if (mem->umem) {
ib_umem_release(mem->umem);
mem->umem = NULL;
static void free_kernel_qp(struct erdma_qp *qp)
{
struct erdma_dev *dev = qp->dev;
vfree(qp->kern_qp.swr_tbl);
vfree(qp->kern_qp.rwr_tbl);
erdma_mem_uninit(dev, &qp->kern_qp.sq_mem);
dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec,
qp->kern_qp.sq_dbrec_dma);
erdma_mem_uninit(dev, &qp->kern_qp.rq_mem);
dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec,
qp->kern_qp.rq_dbrec_dma);
}
static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
struct ib_qp_init_attr *attrs)
{
struct erdma_mem_init_attr attr = {
.type = ERDMA_KMEM,
};
struct erdma_kqp *kqp = &qp->kern_qp;
int ret = -ENOMEM;
int size;
if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
kqp->sig_all = 1;
kqp->sq_pi = 0;
kqp->sq_ci = 0;
kqp->rq_pi = 0;
kqp->rq_ci = 0;
kqp->hw_sq_db =
dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
if (!kqp->swr_tbl)
return -ENOMEM;
kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
if (!kqp->rwr_tbl)
goto err_free_swr_tbl;
size = qp->attrs.sq_size << SQEBB_SHIFT;
attr.len = size;
ret = erdma_mem_init(dev, &kqp->sq_mem, &attr);
if (ret)
goto err_free_rwr_tbl;
kqp->sq_dbrec =
dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma);
if (!kqp->sq_dbrec) {
ret = -ENOMEM;
goto err_free_sq_mem;
}
size = qp->attrs.rq_size << RQE_SHIFT;
attr.len = size;
ret = erdma_mem_init(dev, &kqp->rq_mem, &attr);
if (ret)
goto err_free_sq_dbrec;
kqp->rq_dbrec =
dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma);
if (!kqp->rq_dbrec) {
ret = -ENOMEM;
goto err_free_rq_mem;
}
return 0;
err_free_rq_mem:
erdma_mem_uninit(dev, &kqp->rq_mem);
err_free_sq_dbrec:
dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma);
err_free_sq_mem:
erdma_mem_uninit(dev, &kqp->sq_mem);
err_free_rwr_tbl:
vfree(kqp->rwr_tbl);
err_free_swr_tbl:
vfree(kqp->swr_tbl);
return ret;
}
static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
@ -941,6 +999,9 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
u64 va, u32 len, u64 dbrec_va)
{
struct erdma_mem_init_attr attr = {
.type = ERDMA_UMEM,
};
dma_addr_t dbrec_dma;
u32 rq_offset;
int ret;
@ -949,45 +1010,47 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
qp->attrs.rq_size * RQE_SIZE))
return -EINVAL;
ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
(SZ_1M - SZ_4K), true);
attr.start = va;
attr.virt = va;
attr.len = qp->attrs.sq_size << SQEBB_SHIFT;
attr.req_page_size = SZ_1M - SZ_4K;
ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, &attr);
if (ret)
return ret;
rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
qp->user_qp.rq_offset = rq_offset;
ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
(SZ_1M - SZ_4K), true);
attr.start = va + rq_offset;
attr.virt = va + rq_offset;
attr.len = qp->attrs.rq_size << RQE_SHIFT;
ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, &attr);
if (ret)
goto put_sq_mtt;
goto uninit_sq_mem;
ret = erdma_map_user_dbrecords(uctx, dbrec_va,
&qp->user_qp.user_dbr_page,
&dbrec_dma);
&qp->user_qp.user_dbr_page, &dbrec_dma);
if (ret)
goto put_rq_mtt;
goto uninit_rq_mem;
qp->user_qp.sq_dbrec_dma = dbrec_dma;
qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
return 0;
put_rq_mtt:
put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
uninit_rq_mem:
erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
put_sq_mtt:
put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
uninit_sq_mem:
erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
return ret;
}
static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
{
put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
}
@ -1234,8 +1297,11 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
u64 virt, int access, struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct erdma_mr *mr = NULL;
struct erdma_dev *dev = to_edev(ibpd->device);
struct erdma_mem_init_attr attr = {
.type = ERDMA_UMEM,
};
struct erdma_mr *mr = NULL;
u32 stag;
int ret;
@ -1249,8 +1315,13 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
if (!mr)
return ERR_PTR(-ENOMEM);
ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
SZ_2G - SZ_4K, false);
attr.start = start;
attr.virt = virt;
attr.len = len;
attr.req_page_size = SZ_2G - SZ_4K;
attr.access = access;
attr.flags = ERDMA_MEM_FLAG_MR_BUF;
ret = erdma_mem_init(dev, &mr->mem, &attr);
if (ret)
goto err_out_free;
@ -1260,8 +1331,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
mr->ibmr.lkey = mr->ibmr.rkey = stag;
mr->ibmr.pd = ibpd;
mr->mem.va = virt;
mr->mem.len = len;
mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
mr->valid = 1;
mr->type = ERDMA_MR_TYPE_NORMAL;
@ -1277,7 +1346,7 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
mr->ibmr.lkey >> 8);
err_out_put_mtt:
put_mtt_entries(dev, &mr->mem);
erdma_mem_uninit(dev, &mr->mem);
err_out_free:
kfree(mr);
@ -1314,7 +1383,8 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
put_mtt_entries(dev, &mr->mem);
if (mr->type != ERDMA_MR_TYPE_DMA)
erdma_mem_uninit(dev, &mr->mem);
kfree(mr);
return 0;
@ -1349,13 +1419,12 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
wait_for_completion(&cq->free);
if (rdma_is_kernel_res(&cq->ibcq.res)) {
dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
cq->kern_cq.dbrec_dma);
} else {
erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
}
return 0;
@ -1407,8 +1476,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
if (rdma_is_kernel_res(&qp->ibqp.res)) {
free_kernel_qp(qp);
} else {
put_mtt_entries(dev, &qp->user_qp.sq_mem);
put_mtt_entries(dev, &qp->user_qp.rq_mem);
erdma_mem_uninit(dev, &qp->user_qp.sq_mem);
erdma_mem_uninit(dev, &qp->user_qp.rq_mem);
erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
}
@ -1926,12 +1995,19 @@ int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
struct erdma_ureq_create_cq *ureq)
{
int ret;
struct erdma_dev *dev = to_edev(cq->ibcq.device);
struct erdma_mem_init_attr attr = {
.type = ERDMA_UMEM,
};
int ret;
ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
true);
attr.start = ureq->qbuf_va;
attr.virt = ureq->qbuf_va;
attr.len = ureq->qbuf_len;
attr.req_page_size = SZ_64M - SZ_4K;
attr.access = IB_ACCESS_LOCAL_WRITE;
attr.flags = ERDMA_MEM_FLAG_CQ_BUF;
ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr);
if (ret)
return ret;
@ -1939,7 +2015,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
&cq->user_cq.user_dbr_page,
&cq->user_cq.dbrec_dma);
if (ret)
put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
return ret;
}
@ -1947,12 +2023,15 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
static int erdma_init_kernel_cq(struct erdma_cq *cq)
{
struct erdma_dev *dev = to_edev(cq->ibcq.device);
struct erdma_mem_init_attr attr = {
.type = ERDMA_KMEM,
.len = cq->depth << CQE_SHIFT,
};
int ret;
cq->kern_cq.qbuf =
dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
&cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
if (!cq->kern_cq.qbuf)
return -ENOMEM;
ret = erdma_mem_init(dev, &cq->kern_cq.qbuf_mem, &attr);
if (ret)
return ret;
cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL,
&cq->kern_cq.dbrec_dma);
@ -1966,8 +2045,7 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq)
return 0;
err_out:
dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT,
cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
return -ENOMEM;
}
@ -2032,10 +2110,9 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
err_free_res:
if (!rdma_is_kernel_res(&ibcq->res)) {
erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
} else {
dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem);
dma_pool_free(dev->db_pool, cq->kern_cq.dbrec,
cq->kern_cq.dbrec_dma);
}

66
drivers/infiniband/hw/erdma/erdma_verbs.h
View File

@ -111,19 +111,71 @@ struct erdma_mtt {
struct erdma_mtt *low_level;
};
enum erdma_mem_type {
ERDMA_NO_MEM = 0,
ERDMA_UMEM = 1,
ERDMA_KMEM = 2,
};
struct erdma_buf_list {
void *buf;
dma_addr_t dma_addr;
};
struct erdma_kmem {
struct erdma_buf_list *buf_list;
};
enum erdma_mem_flags {
ERDMA_MEM_FLAG_MR_BUF = (1 << 0),
ERDMA_MEM_FLAG_CQ_BUF = (1 << 1),
};
struct erdma_mem_init_attr {
enum erdma_mem_type type;
u64 start;
u64 virt;
u64 len;
unsigned long req_page_size;
int access;
u32 flags;
};
struct erdma_mem {
struct ib_umem *umem;
struct erdma_mtt *mtt;
enum erdma_mem_type type;
union {
struct ib_umem *umem;
struct erdma_kmem kmem;
};
u32 page_size;
u32 page_offset;
u32 page_cnt;
u32 mtt_nents;
struct erdma_mtt *mtt;
u64 va;
u64 len;
};
static inline void *erdma_kmem_get_entry(struct erdma_mem *mem, u32 idx,
u32 depth, u32 shift)
{
u32 offset = (idx & (depth - 1)) << shift;
return (u8 *)mem->kmem.buf_list[offset >> PAGE_SHIFT].buf +
offset_in_page(offset);
}
static inline void erdma_kmem_clear(struct erdma_mem *mem)
{
u32 i;
for (i = 0; i < mem->page_cnt; i++)
memset(mem->kmem.buf_list[i].buf, 0, PAGE_SIZE);
}
struct erdma_mr {
struct ib_mr ibmr;
struct erdma_mem mem;
@ -183,11 +235,8 @@ struct erdma_kqp {
void __iomem *hw_sq_db;
void __iomem *hw_rq_db;
void *sq_buf;
dma_addr_t sq_buf_dma_addr;
void *rq_buf;
dma_addr_t rq_buf_dma_addr;
struct erdma_mem sq_mem;
struct erdma_mem rq_mem;
void *sq_dbrec;
void *rq_dbrec;
@ -320,8 +369,7 @@ struct erdma_qp {
};
struct erdma_kcq_info {
void *qbuf;
dma_addr_t qbuf_dma_addr;
struct erdma_mem qbuf_mem;
u32 ci;
u32 cmdsn;
u32 notify_cnt;

2
drivers/infiniband/hw/hfi1/affinity.c
View File

@ -386,7 +386,7 @@ static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
/*
* This function creates the table for looking up CPUs for completion vectors.
* num_comp_vectors needs to have been initilized before calling this function.
* num_comp_vectors needs to have been initialized before calling this function.
*/
static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
struct hfi1_affinity_node *entry)

4
drivers/infiniband/hw/hfi1/chip.c
View File

@ -5567,7 +5567,7 @@ static int init_rcverr(struct hfi1_devdata *dd)
static void free_rcverr(struct hfi1_devdata *dd)
{
if (dd->rcverr_timer.function)
timer_delete_sync(&dd->rcverr_timer);
timer_shutdown_sync(&dd->rcverr_timer);
}
static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg)
@ -12300,7 +12300,7 @@ static void free_cntrs(struct hfi1_devdata *dd)
int i;
if (dd->synth_stats_timer.function)
timer_delete_sync(&dd->synth_stats_timer);
timer_shutdown_sync(&dd->synth_stats_timer);
cancel_work_sync(&dd->update_cntr_work);
ppd = (struct hfi1_pportdata *)(dd + 1);
for (i = 0; i < dd->num_pports; i++, ppd++) {

2
drivers/infiniband/hw/hfi1/firmware.c
View File

@ -77,7 +77,7 @@ struct css_header {
/* size of platform configuration partition */
#define MAX_PLATFORM_CONFIG_FILE_SIZE 4096
/* size of file of plaform configuration encoded in format version 4 */
/* size of file of platform configuration encoded in format version 4 */
#define PLATFORM_CONFIG_FORMAT_4_FILE_SIZE 528
/* the file itself */

14
drivers/infiniband/hw/hfi1/pcie.c
View File

@ -53,17 +53,9 @@ int hfi1_pcie_init(struct pci_dev *pdev)
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
if (ret) {
/*
* If the 64 bit setup fails, try 32 bit. Some systems
* do not setup 64 bit maps on systems with 2GB or less
* memory installed.
*/
ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (ret) {
dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n",
ERR_PTR(ret));
goto bail;
}
dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n",
ERR_PTR(ret));
goto bail;
}
pci_set_master(pdev);

3
drivers/infiniband/hw/hfi1/sdma.c
View File

@ -1569,6 +1569,9 @@ void sdma_exit(struct hfi1_devdata *dd)
* it is not running.
*/
sdma_finalput(&sde->state);
cancel_work_sync(&sde->err_halt_worker);
tasklet_kill(&sde->sdma_hw_clean_up_task);
cancel_work_sync(&sde->flush_worker);
}
}

4
drivers/infiniband/hw/hfi1/verbs.c
View File

@ -1888,6 +1888,10 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd)
void hfi1_unregister_ib_device(struct hfi1_devdata *dd)
{
struct hfi1_ibdev *dev = &dd->verbs_dev;
int i;
for (i = 0; i < dd->num_pports; i++)
timer_shutdown_sync(&dd->pport[i].ibport_data.rvp.trap_timer);
hfi1_verbs_unregister_sysfs(dd);

4
drivers/infiniband/hw/hns/hns_roce_bond.c
View File

@ -81,7 +81,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp,
{
struct net_device *active_dev;
struct net_device *old_dev;
int i, ret = 0;
int i = 0, ret = 0;
if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
rcu_read_lock();
@ -89,7 +89,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp,
bond_option_active_slave_get_rcu(netdev_priv(bond_grp->upper_dev));
rcu_read_unlock();
} else {
for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) {
for (; i < ROCE_BOND_FUNC_MAX; i++) {
active_dev = bond_grp->bond_func_info[i].net_dev;
if (active_dev &&
ib_get_curr_port_state(active_dev) == IB_PORT_ACTIVE)

3
drivers/infiniband/hw/hns/hns_roce_cq.c
View File

@ -261,10 +261,11 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,
buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size;
buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num;
buf_attr.region_count = 1;
buf_attr.user_access = IB_ACCESS_LOCAL_WRITE;
ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,
hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT,
udata, addr);
udata, addr, true);
if (ret)
ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret);

55
drivers/infiniband/hw/hns/hns_roce_debugfs.c
View File

@ -338,6 +338,60 @@ static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev,
}
}
static int gsi_sl_debugfs_show(struct seq_file *file, void *offset)
{
struct hns_roce_dev *hr_dev = file->private;
seq_printf(file, "%u\n", hr_dev->gsi_sl);
return 0;
}
static ssize_t gsi_sl_debugfs_store(char *buf, size_t count, void *data)
{
struct hns_roce_dev *hr_dev = data;
struct net_device *netdev;
int ret;
u8 val;
netdev = ib_device_get_netdev(&hr_dev->ib_dev, 1);
if (!netdev)
return -ENODEV;
if (ib_get_curr_port_state(netdev) == IB_PORT_ACTIVE) {
ret = -EOPNOTSUPP;
goto out;
}
ret = kstrtou8(buf, 0, &val);
if (ret)
goto out;
if (!check_sl_valid(hr_dev, val)) {
ret = -EINVAL;
goto out;
}
hr_dev->gsi_sl = val;
out:
dev_put(netdev);
return ret ? : count;
}
static void create_gsi_sl_debugfs(struct hns_roce_dev *hr_dev,
struct dentry *parent)
{
struct hns_debugfs_seqfile *seqfile = &hr_dev->dbgfs.gsi_sl;
seqfile->read = gsi_sl_debugfs_show;
seqfile->write = gsi_sl_debugfs_store;
seqfile->data = hr_dev;
debugfs_create_file("gsi_sl", 0600, parent, seqfile,
&hns_debugfs_seqfile_fops);
}
/* debugfs for device */
void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)
{
@ -348,6 +402,7 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)
create_sw_stat_debugfs(hr_dev, dbgfs->root);
create_cc_param_debugfs(hr_dev, dbgfs->root);
create_gsi_sl_debugfs(hr_dev, dbgfs->root);
}
void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev)

1
drivers/infiniband/hw/hns/hns_roce_debugfs.h
View File

@ -47,6 +47,7 @@ struct hns_roce_dev_debugfs {
struct dentry *root;
struct hns_sw_stat_debugfs sw_stat_root;
struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM];
struct hns_debugfs_seqfile gsi_sl;
};
struct hns_roce_dev;

6
drivers/infiniband/hw/hns/hns_roce_device.h
View File

@ -654,6 +654,7 @@ struct hns_roce_qp {
u8 priority;
spinlock_t flush_lock;
struct hns_roce_dip *dip;
bool ud_sl_set;
};
struct hns_roce_ib_iboe {
@ -737,7 +738,7 @@ enum hns_roce_scc_algo {
struct hns_roce_caps {
u64 fw_ver;
u8 num_ports;
int gid_table_len[HNS_ROCE_MAX_PORTS];
u32 gid_table_len[HNS_ROCE_MAX_PORTS];
int pkey_table_len[HNS_ROCE_MAX_PORTS];
int local_ca_ack_delay;
int num_uars;
@ -1047,6 +1048,7 @@ struct hns_roce_dev {
struct hns_roce_dev_debugfs dbgfs;
atomic64_t *dfx_cnt;
struct hns_roce_scc_param *scc_param;
u8 gsi_sl;
};
enum hns_roce_trace_type {
@ -1230,7 +1232,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
struct hns_roce_buf_attr *buf_attr,
unsigned int page_shift, struct ib_udata *udata,
unsigned long user_addr);
unsigned long user_addr, bool is_cq);
void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev,
struct hns_roce_mtr *mtr);
int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,

63
drivers/infiniband/hw/hns/hns_roce_hw_v2.c
View File

@ -431,7 +431,8 @@ static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
return 0;
}
static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
static int fill_ud_av(struct hns_roce_qp *qp,
struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
struct hns_roce_ah *ah)
{
struct ib_device *ib_dev = ah->ibah.device;
@ -441,7 +442,13 @@ static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe,
hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit);
hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass);
hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel);
hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl);
if (!qp->ud_sl_set || qp->ibqp.qp_type == IB_QPT_GSI) {
qp->sl = qp->ibqp.qp_type == IB_QPT_GSI ?
hr_dev->gsi_sl : ah->av.sl;
qp->ud_sl_set = true;
}
hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, qp->sl);
ud_sq_wqe->sgid_index = ah->av.gid_index;
@ -491,12 +498,10 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
qp->qkey : ud_wr(wr)->remote_qkey);
hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn);
ret = fill_ud_av(ud_sq_wqe, ah);
ret = fill_ud_av(qp, ud_sq_wqe, ah);
if (ret)
return ret;
qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;
set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
/*
@ -2416,8 +2421,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev)
caps->gmv_bt_num *
(HNS_HW_PAGE_SIZE / caps->gmv_entry_sz));
caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE /
caps->gmv_entry_sz);
caps->gmv_entry_num = caps->gid_table_len[0];
} else {
u32 func_num = max_t(u32, 1, hr_dev->func_num);
@ -3824,7 +3828,8 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
* Now backwards through the CQ, removing CQ entries
* that match our QP by overwriting them with next entries.
*/
while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
while (prod_index != hr_cq->cons_index) {
--prod_index;
cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) {
if (srq && hr_reg_read(cqe, CQE_S_R)) {
@ -4353,18 +4358,39 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type,
static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj,
dma_addr_t base_addr)
{
u32 obj_num_per_bt = HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz;
u32 chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + PAGE_SHIFT);
u32 bt_num_per_chunk = chunk_size / HNS_HW_PAGE_SIZE;
u32 first = obj / obj_num_per_bt;
u32 last = min(first + bt_num_per_chunk, hr_dev->caps.gmv_bt_num);
struct hns_roce_cmq_desc desc;
struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data;
u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz);
u64 addr = to_hr_hw_page_addr(base_addr);
struct hns_roce_cmq_req *req;
u64 addr;
int ret;
u32 i;
hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false);
/* The GMV BT entry of hardware covers a fixed 4K region, so a buffer
* chunk larger than 4K must be registered to hardware with one BT
* entry per 4K block, otherwise the GMV entries beyond the first
* 4K of the chunk are unreachable.
*/
for (i = first; i < last; i++) {
hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT,
false);
req = (struct hns_roce_cmq_req *)desc.data;
hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
hr_reg_write(req, CFG_GMV_BT_IDX, idx);
addr = to_hr_hw_page_addr(base_addr +
(u64)(i - first) * HNS_HW_PAGE_SIZE);
hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr));
hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr));
hr_reg_write(req, CFG_GMV_BT_IDX, i);
return hns_roce_cmq_send(hr_dev, &desc, 1);
ret = hns_roce_cmq_send(hr_dev, &desc, 1);
if (ret)
return ret;
}
return 0;
}
static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj,
@ -5327,7 +5353,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp,
hr_reg_clear(qpc_mask, QPC_VLAN_ID);
if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n",
ibdev_err(ibdev, "sgid_index(%u) too large. max is %u\n",
grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]);
return -EINVAL;
}
@ -5603,6 +5629,7 @@ static void v2_set_flushed_fields(struct ib_qp *ibqp,
hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head);
hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX);
hr_qp->state = IB_QPS_ERR;
hr_qp->ud_sl_set = false;
spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag);
if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */
@ -6913,7 +6940,7 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr,
hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL,
0);
0, false);
if (err)
dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err);

22
drivers/infiniband/hw/hns/hns_roce_mr.c
View File

@ -112,7 +112,7 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,
err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr,
hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT,
udata, start);
udata, start, false);
if (err) {
ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err);
return err;
@ -586,7 +586,8 @@ static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
struct hns_roce_buf_attr *buf_attr,
struct ib_udata *udata, unsigned long user_addr)
struct ib_udata *udata, unsigned long user_addr,
bool is_cq)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
size_t total_size;
@ -595,8 +596,14 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
if (udata) {
mtr->kmem = NULL;
mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
buf_attr->user_access);
if (is_cq)
mtr->umem = ib_umem_get_cq_buf_or_va(ibdev, NULL,
user_addr,
total_size,
buf_attr->user_access);
else
mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size,
buf_attr->user_access);
if (IS_ERR(mtr->umem)) {
ibdev_err(ibdev, "failed to get umem, ret = %pe.\n",
mtr->umem);
@ -1035,11 +1042,13 @@ static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)
* @ba_page_shift: page shift for multi-hop base address table
* @udata: user space context, if it's NULL, means kernel space
* @user_addr: userspace virtual address to start at
* @is_cq: true when @mtr backs a CQ buffer, which the device writes
* completion entries into
*/
int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
struct hns_roce_buf_attr *buf_attr,
unsigned int ba_page_shift, struct ib_udata *udata,
unsigned long user_addr)
unsigned long user_addr, bool is_cq)
{
struct ib_device *ibdev = &hr_dev->ib_dev;
int ret;
@ -1052,7 +1061,8 @@ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
mtr->umem = NULL;
mtr->kmem = NULL;
} else {
ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr);
ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr,
is_cq);
if (ret) {
ibdev_err(ibdev,
"failed to alloc mtr bufs, ret = %d.\n", ret);

2
drivers/infiniband/hw/hns/hns_roce_qp.c
View File

@ -808,7 +808,7 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
}
ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
udata, addr);
udata, addr, false);
if (ret) {
ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
goto err_inline;

4
drivers/infiniband/hw/hns/hns_roce_srq.c
View File

@ -180,7 +180,7 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq,
ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr,
hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT,
udata, addr);
udata, addr, false);
if (ret) {
ibdev_err(ibdev,
"failed to alloc SRQ idx mtr, ret = %d.\n", ret);
@ -235,7 +235,7 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev,
ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr,
hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT,
udata, addr);
udata, addr, false);
if (ret)
ibdev_err(ibdev,
"failed to alloc SRQ buf mtr, ret = %d.\n", ret);

117
drivers/infiniband/hw/ionic/ionic_admin.c
View File

@ -489,6 +489,49 @@ static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev,
return ionic_rdma_devcmd(dev, &admin);
}
static int ionic_rdma_cq_devcmd(struct ionic_vcq *vcq,
struct ionic_queue *q,
u32 *qid, u32 cid, u16 opcode,
u8 udma_idx)
{
struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device);
struct ionic_admin_ctx admin = {
.work = COMPLETION_INITIALIZER_ONSTACK(admin.work),
.cmd.rdma_queue = {
.opcode = opcode,
.lif_index = cpu_to_le16(dev->lif_cfg.lif_index),
.qid_ver = cpu_to_le32(*qid),
.cid = cpu_to_le32(cid),
.dbid = cpu_to_le16(dev->lif_cfg.dbid),
.depth_log2 = q->depth_log2,
.stride_log2 = q->stride_log2,
.dma_addr = cpu_to_le64(q->dma),
},
};
int rc;
if (ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) {
struct ionic_admin_create_cq_resp resp_buf;
/* Indicates use of CQ create V2 response format. */
admin.cmd.rdma_queue.qid_ver = cpu_to_le32(IONIC_CREATE_CQ_CMD_V2_MAGIC);
admin.cmd.rdma_queue.udma_idx = udma_idx;
rc = ionic_rdma_devcmd(dev, &admin);
if (rc)
return rc;
memcpy(&resp_buf, admin.comp.comp.cmd_data, sizeof(resp_buf));
*qid = le32_to_cpu(resp_buf.id);
} else {
rc = ionic_rdma_devcmd(dev, &admin);
if (rc)
return rc;
}
return 0;
}
static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context)
{
struct ionic_aq *aq = cq_context;
@ -533,12 +576,16 @@ static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev,
cq = &vcq->cq[0];
rc = ionic_create_cq_common(vcq, &buf, &attr, NULL, NULL,
NULL, NULL, 0);
NULL, 0);
if (rc)
goto err_init;
rc = ionic_rdma_queue_devcmd(dev, &cq->q, cq->cqid, cq->eqid,
IONIC_CMD_RDMA_CREATE_CQ);
rc = ionic_rdma_cq_devcmd(vcq, &cq->q, &cq->cqid, cq->eqid,
IONIC_CMD_RDMA_CREATE_CQ, 0);
if (rc)
goto err_cmd;
rc = ionic_post_create_cq_cmd(cq, NULL, NULL);
if (rc)
goto err_cmd;
@ -552,6 +599,14 @@ static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev,
return ERR_PTR(rc);
}
static void ionic_destroy_rdma_admincq(struct ionic_ibdev *dev,
struct ionic_vcq *vcq)
{
ionic_pre_destroy_cq_cmd(dev, &vcq->cq[0]);
ionic_destroy_cq_common(dev, &vcq->cq[0]);
kfree(vcq);
}
static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev,
u32 aqid, u32 cqid)
{
@ -890,6 +945,49 @@ static void ionic_qp_event(struct ionic_ibdev *dev, u32 qpid, u8 code)
kref_put(&qp->qp_kref, ionic_qp_complete);
}
static void ionic_srq_event(struct ionic_ibdev *dev, u32 srqid, u8 code)
{
unsigned long irqflags;
struct ionic_srq *srq;
struct ib_event ibev;
xa_lock_irqsave(&dev->srq_tbl, irqflags);
srq = xa_load(&dev->srq_tbl, srqid);
if (srq)
kref_get(&srq->kref);
xa_unlock_irqrestore(&dev->srq_tbl, irqflags);
if (!srq) {
ibdev_dbg(&dev->ibdev,
"missing srqid %#x code %u\n", srqid, code);
return;
}
ibev.device = &dev->ibdev;
ibev.element.srq = &srq->ibsrq;
switch (code) {
case IONIC_V1_EQE_SRQ_LIMIT_REACHED:
ibev.event = IB_EVENT_SRQ_LIMIT_REACHED;
break;
case IONIC_V1_EQE_SRQ_ERR:
ibev.event = IB_EVENT_SRQ_ERR;
break;
default:
ibdev_dbg(&dev->ibdev,
"unrecognized srqid %#x code %u\n", srqid, code);
goto out;
}
if (srq->ibsrq.event_handler)
srq->ibsrq.event_handler(&ibev, srq->ibsrq.srq_context);
out:
kref_put(&srq->kref, ionic_srq_complete);
}
static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget)
{
struct ionic_ibdev *dev = eq->dev;
@ -923,6 +1021,10 @@ static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget)
ionic_qp_event(dev, qid, code);
break;
case IONIC_V1_EQE_TYPE_SRQ:
ionic_srq_event(dev, qid, code);
break;
default:
ibdev_dbg(&dev->ibdev,
"unknown event %#x type %u\n", evt, type);
@ -1153,8 +1255,7 @@ int ionic_create_rdma_admin(struct ionic_ibdev *dev)
vcq->cq[0].cqid);
if (IS_ERR(aq)) {
/* Clean up the dangling CQ */
ionic_destroy_cq_common(dev, &vcq->cq[0]);
kfree(vcq);
ionic_destroy_rdma_admincq(dev, vcq);
rc = PTR_ERR(aq);
@ -1207,10 +1308,8 @@ void ionic_destroy_rdma_admin(struct ionic_ibdev *dev)
cancel_work_sync(&aq->work);
__ionic_destroy_rdma_adminq(dev, aq);
if (vcq) {
ionic_destroy_cq_common(dev, &vcq->cq[0]);
kfree(vcq);
}
if (vcq)
ionic_destroy_rdma_admincq(dev, vcq);
}
kfree(dev->aq_vec);

830
drivers/infiniband/hw/ionic/ionic_controlpath.c
View File

File diff suppressed because it is too large Load Diff

98
drivers/infiniband/hw/ionic/ionic_datapath.c
View File

@ -34,17 +34,17 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
struct ionic_v1_wqe *wqe;
u64 wqe_idx;
if (!qp->rq_flush)
if (!qp->rq.flush)
return 0;
if (ionic_queue_empty(&qp->rq))
if (ionic_queue_empty(&qp->rq.q))
return 0;
wqe = ionic_queue_at_cons(&qp->rq);
wqe = ionic_queue_at_cons(&qp->rq.q);
wqe_idx = le64_to_cpu(wqe->base.wqe_idx);
/* wqe_idx must be a valid queue index */
if (unlikely(wqe_idx >> qp->rq.depth_log2)) {
if (unlikely(wqe_idx >> qp->rq.q.depth_log2)) {
ibdev_warn(qp->ibqp.device,
"flush qp %u recv index %llu invalid\n",
qp->qpid, (unsigned long long)wqe_idx);
@ -52,7 +52,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
}
/* wqe_idx must indicate a request that is outstanding */
meta = &qp->rq_meta[wqe_idx];
meta = &qp->rq.meta[wqe_idx];
if (unlikely(meta->next != IONIC_META_POSTED)) {
ibdev_warn(qp->ibqp.device,
"flush qp %u recv index %llu not posted\n",
@ -60,7 +60,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
return -EIO;
}
ionic_queue_consume(&qp->rq);
ionic_queue_consume(&qp->rq.q);
memset(wc, 0, sizeof(*wc));
@ -68,8 +68,8 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc)
wc->wr_id = meta->wrid;
wc->qp = &qp->ibqp;
meta->next = qp->rq_meta_head;
qp->rq_meta_head = meta;
meta->next = qp->rq.meta_head;
qp->rq.meta_head = meta;
return 1;
}
@ -139,7 +139,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
u32 src_qpn, st_len;
u8 op;
if (cqe_qp->rq_flush)
if (cqe_qp->rq.flush)
return 0;
qp = cqe_qp;
@ -148,38 +148,38 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
/* ignore wqe_idx in case of flush error */
if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) {
cqe_qp->rq_flush = true;
cqe_qp->rq.flush = true;
cq->flush = true;
list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
list_move_tail(&qp->rq.cq_flush, &cq->flush_rq);
/* posted recvs (if any) flushed by ionic_flush_recv */
return 0;
}
/* there had better be something in the recv queue to complete */
if (ionic_queue_empty(&qp->rq)) {
if (ionic_queue_empty(&qp->rq.q)) {
ibdev_warn(&dev->ibdev, "qp %u is empty\n", qp->qpid);
return -EIO;
}
wqe_idx = le64_to_cpu(cqe->recv.wqe_idx_timestamp) & IONIC_V1_CQE_WQE_IDX_MASK;
/* wqe_idx must be a valid queue index */
if (unlikely(wqe_idx >> qp->rq.depth_log2)) {
if (unlikely(wqe_idx >> qp->rq.q.depth_log2)) {
ibdev_warn(&dev->ibdev,
"qp %u recv index %u invalid\n", qp->qpid, wqe_idx);
return -EIO;
}
/* wqe_idx must indicate a request that is outstanding */
meta = &qp->rq_meta[wqe_idx];
meta = &qp->rq.meta[wqe_idx];
if (unlikely(meta->next != IONIC_META_POSTED)) {
ibdev_warn(&dev->ibdev,
"qp %u recv index %u not posted\n", qp->qpid, wqe_idx);
return -EIO;
}
meta->next = qp->rq_meta_head;
qp->rq_meta_head = meta;
meta->next = qp->rq.meta_head;
qp->rq.meta_head = meta;
memset(wc, 0, sizeof(*wc));
@ -191,9 +191,9 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
wc->vendor_err = st_len;
wc->status = ionic_to_ib_status(st_len);
cqe_qp->rq_flush = true;
cqe_qp->rq.flush = true;
cq->flush = true;
list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
list_move_tail(&qp->rq.cq_flush, &cq->flush_rq);
ibdev_warn(&dev->ibdev,
"qp %d recv cqe with error\n", qp->qpid);
@ -255,7 +255,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq,
wc->port_num = 1;
out:
ionic_queue_consume(&qp->rq);
ionic_queue_consume(&qp->rq.q);
return 1;
}
@ -526,9 +526,15 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev,
switch (type) {
case IONIC_V1_CQE_TYPE_RECV:
spin_lock(&qp->rq_lock);
if (qp->srq) {
ibdev_dbg(&dev->ibdev,
"srq recv cqe for qp %u\n", qid);
goto cq_next;
}
spin_lock(&qp->rq.lock);
rc = ionic_poll_recv(dev, cq, qp, cqe, wc + npolled);
spin_unlock(&qp->rq_lock);
spin_unlock(&qp->rq.lock);
if (rc < 0)
goto out;
@ -612,19 +618,19 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev,
cq->flush = true;
}
list_for_each_entry_safe(qp, qp_next, &cq->flush_rq, cq_flush_rq) {
list_for_each_entry_safe(qp, qp_next, &cq->flush_rq, rq.cq_flush) {
if (npolled == nwc)
goto out;
spin_lock(&qp->rq_lock);
spin_lock(&qp->rq.lock);
rc = ionic_flush_recv_many(qp, wc + npolled, nwc - npolled);
spin_unlock(&qp->rq_lock);
spin_unlock(&qp->rq.lock);
if (rc > 0)
npolled += rc;
if (npolled < nwc)
list_del_init(&qp->cq_flush_rq);
list_del_init(&qp->rq.cq_flush);
else
cq->flush = true;
}
@ -881,7 +887,7 @@ static void ionic_prep_sq_wqe(struct ionic_qp *qp, void *wqe)
static void ionic_prep_rq_wqe(struct ionic_qp *qp, void *wqe)
{
memset(wqe, 0, 1u << qp->rq.stride_log2);
memset(wqe, 0, 1u << qp->rq.q.stride_log2);
}
static int ionic_prep_send(struct ionic_qp *qp,
@ -1183,38 +1189,38 @@ static int ionic_prep_recv(struct ionic_qp *qp,
s64 signed_len;
u32 mval;
wqe = ionic_queue_at_prod(&qp->rq);
wqe = ionic_queue_at_prod(&qp->rq.q);
/* if wqe is owned by device, caller can try posting again soon */
if (wqe->base.flags & cpu_to_be16(IONIC_V1_FLAG_FENCE))
return -EAGAIN;
meta = qp->rq_meta_head;
meta = qp->rq.meta_head;
if (unlikely(meta == IONIC_META_LAST) ||
unlikely(meta == IONIC_META_POSTED))
return -EIO;
ionic_prep_rq_wqe(qp, wqe);
mval = ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, qp->rq_spec,
mval = ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, qp->rq.spec,
false);
signed_len = ionic_prep_pld(wqe, &wqe->recv.pld,
qp->rq_spec, mval,
qp->rq.spec, mval,
wr->sg_list, wr->num_sge);
if (signed_len < 0)
return signed_len;
meta->wrid = wr->wr_id;
wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq_meta);
wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq.meta);
wqe->base.num_sge_key = wr->num_sge;
/* total length for recv goes in base imm_data_key */
wqe->base.imm_data_key = cpu_to_be32(signed_len);
ionic_queue_produce(&qp->rq);
ionic_queue_produce(&qp->rq.q);
qp->rq_meta_head = meta->next;
qp->rq.meta_head = meta->next;
meta->next = IONIC_META_POSTED;
return 0;
@ -1311,7 +1317,7 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev,
if (!bad)
return -EINVAL;
if (!qp->has_rq) {
if (qp->srq) {
*bad = wr;
return -EINVAL;
}
@ -1321,10 +1327,10 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev,
return -EINVAL;
}
spin_lock_irqsave(&qp->rq_lock, irqflags);
spin_lock_irqsave(&qp->rq.lock, irqflags);
while (wr) {
if (ionic_queue_full(&qp->rq)) {
if (ionic_queue_full(&qp->rq.q)) {
ibdev_dbg(&dev->ibdev, "queue full");
rc = -ENOMEM;
goto out;
@ -1339,32 +1345,32 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev,
out:
if (!cq) {
spin_unlock_irqrestore(&qp->rq_lock, irqflags);
spin_unlock_irqrestore(&qp->rq.lock, irqflags);
goto out_unlocked;
}
spin_unlock_irqrestore(&qp->rq_lock, irqflags);
spin_unlock_irqrestore(&qp->rq.lock, irqflags);
spin_lock_irqsave(&cq->lock, irqflags);
spin_lock(&qp->rq_lock);
spin_lock(&qp->rq.lock);
if (likely(qp->rq.prod != qp->rq_old_prod)) {
if (likely(qp->rq.q.prod != qp->rq.old_prod)) {
/* ring cq doorbell just in time */
spend = (qp->rq.prod - qp->rq_old_prod) & qp->rq.mask;
spend = (qp->rq.q.prod - qp->rq.old_prod) & qp->rq.q.mask;
ionic_reserve_cq(dev, cq, spend);
qp->rq_old_prod = qp->rq.prod;
qp->rq.old_prod = qp->rq.q.prod;
ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.rq_qtype,
ionic_queue_dbell_val(&qp->rq));
ionic_queue_dbell_val(&qp->rq.q));
}
if (qp->rq_flush) {
if (qp->rq.flush) {
notify = true;
cq->flush = true;
list_move_tail(&qp->cq_flush_rq, &cq->flush_rq);
list_move_tail(&qp->rq.cq_flush, &cq->flush_rq);
}
spin_unlock(&qp->rq_lock);
spin_unlock(&qp->rq.lock);
spin_unlock_irqrestore(&cq->lock, irqflags);
if (notify && vcq->ibcq.comp_handler)

76
drivers/infiniband/hw/ionic/ionic_fw.h
View File

@ -88,6 +88,10 @@ static inline int to_ionic_mr_flags(int access)
return flags;
}
enum ionic_srq_flags {
IONIC_SRQF_CMB = BIT(0),
};
enum ionic_qp_flags {
/* bits that determine qp access */
IONIC_QPF_REMOTE_WRITE = BIT(0),
@ -105,6 +109,8 @@ enum ionic_qp_flags {
IONIC_QPF_SQ_CMB = BIT(13),
IONIC_QPF_RQ_CMB = BIT(14),
IONIC_QPF_PRIVILEGED = BIT(15),
IONIC_QP_USER_FLAGS_MASK = GENMASK(31, 16),
};
static inline int from_ionic_qp_flags(int flags)
@ -744,11 +750,22 @@ struct ionic_admin_create_cq {
__le32 map_count;
__le64 dma_addr;
__le16 dbid_flags;
__u8 udma_idx;
__le32 resp_buf_len;
__le64 resp_dma_addr;
} __packed;
#define IONIC_ADMIN_CREATE_CQ_IN_V1_LEN 34
static_assert(sizeof(struct ionic_admin_create_cq) ==
IONIC_ADMIN_CREATE_CQ_IN_V1_LEN);
#define IONIC_ADMIN_CREATE_CQ_IN_V2_LEN 47
static_assert(sizeof(struct ionic_admin_create_cq) == IONIC_ADMIN_CREATE_CQ_IN_V2_LEN);
struct ionic_admin_create_cq_resp {
__le32 id;
__u8 udma_idx;
} __packed;
#define IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN 5
static_assert(sizeof(struct ionic_admin_create_cq_resp) == IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN);
struct ionic_admin_destroy_cq {
__le32 cq_id;
@ -844,6 +861,48 @@ struct ionic_admin_query_qp {
static_assert(sizeof(struct ionic_admin_query_qp) ==
IONIC_ADMIN_QUERY_QP_IN_V1_LEN);
struct ionic_admin_create_srq {
__le64 dma_addr;
__le32 map_count;
__le32 pd_id;
__le32 qid;
__le16 dbid;
__le16 low_wqes_limit;
__le16 flags;
__u8 depth_log2;
__u8 stride_log2;
__u8 page_size_log2;
__u8 udma_mask;
__le32 resp_buf_len;
__le64 resp_dma_addr;
} __packed;
#define IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN 42
static_assert(sizeof(struct ionic_admin_create_srq) == IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN);
struct ionic_admin_create_srq_resp {
__le32 id;
__u8 udma_idx;
} __packed;
#define IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN 5
static_assert(sizeof(struct ionic_admin_create_srq_resp) == IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN);
struct ionic_admin_modify_srq {
__le32 qid;
__le16 low_wqes_limit;
} __packed;
#define IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN 6
static_assert(sizeof(struct ionic_admin_modify_srq) == IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN);
struct ionic_admin_destroy_srq {
__le32 qid;
} __packed;
#define IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN 4
static_assert(sizeof(struct ionic_admin_destroy_srq) == IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN);
#define ADMIN_WQE_STRIDE 64
#define ADMIN_WQE_HDR_LEN 4
@ -866,6 +925,9 @@ struct ionic_v1_admin_wqe {
struct ionic_admin_destroy_qp destroy_qp;
struct ionic_admin_mod_qp mod_qp;
struct ionic_admin_query_qp query_qp;
struct ionic_admin_create_srq create_srq;
struct ionic_admin_modify_srq modify_srq;
struct ionic_admin_destroy_srq destroy_srq;
} cmd;
};
@ -912,6 +974,10 @@ enum ionic_v1_admin_op {
IONIC_V1_ADMIN_DESTROY_AH,
IONIC_V1_ADMIN_QP_STATS_HDRS,
IONIC_V1_ADMIN_QP_STATS_VALS,
IONIC_V1_ADMIN_CREATE_SRQ = 26,
IONIC_V1_ADMIN_MODIFY_SRQ,
IONIC_V1_ADMIN_QUERY_SRQ,
IONIC_V1_ADMIN_DESTROY_SRQ,
IONIC_V1_ADMIN_OPCODES_MAX,
};
@ -947,7 +1013,7 @@ enum ionic_v1_eqe_evt_bits {
/* cq error events */
IONIC_V1_EQE_CQ_ERR = 8,
/* qp and srq events */
/* qp events */
IONIC_V1_EQE_TYPE_QP = 1,
/* qp normal events */
IONIC_V1_EQE_SRQ_LEVEL = 0,
@ -958,6 +1024,10 @@ enum ionic_v1_eqe_evt_bits {
IONIC_V1_EQE_QP_ERR = 8,
IONIC_V1_EQE_QP_ERR_REQUEST = 9,
IONIC_V1_EQE_QP_ERR_ACCESS = 10,
/* srq events */
IONIC_V1_EQE_TYPE_SRQ = 2,
IONIC_V1_EQE_SRQ_LIMIT_REACHED = 0,
IONIC_V1_EQE_SRQ_ERR = 1,
};
enum ionic_tfp_csum_profiles {

24
drivers/infiniband/hw/ionic/ionic_ibdev.c
View File

@ -73,6 +73,11 @@ static int ionic_query_device(struct ib_device *ibdev,
attr->max_ah = dev->lif_cfg.nahs_per_lif;
attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2;
attr->max_pkeys = IONIC_PKEY_TBL_LEN;
if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) {
attr->max_srq = dev->lif_cfg.srq_count;
attr->max_srq_wr = IONIC_MAX_SRQ_DEPTH;
attr->max_srq_sge = IONIC_MAX_SRQ_SGES;
}
return 0;
}
@ -266,6 +271,15 @@ static const struct ib_device_ops ionic_dev_ops = {
INIT_RDMA_OBJ_SIZE(ib_mw, ionic_mr, ibmw),
};
static const struct ib_device_ops ionic_srq_ops = {
.create_srq = ionic_create_srq,
.modify_srq = ionic_modify_srq,
.query_srq = ionic_query_srq,
.destroy_srq = ionic_destroy_srq,
INIT_RDMA_OBJ_SIZE(ib_srq, ionic_srq, ibsrq),
};
static void ionic_init_resids(struct ionic_ibdev *dev)
{
ionic_resid_init(&dev->inuse_cqid, dev->lif_cfg.cq_count);
@ -304,6 +318,8 @@ static void ionic_destroy_ibdev(struct ionic_ibdev *dev)
xa_destroy(&dev->qp_tbl);
WARN_ON(!xa_empty(&dev->cq_tbl));
xa_destroy(&dev->cq_tbl);
WARN_ON(!xa_empty(&dev->srq_tbl));
xa_destroy(&dev->srq_tbl);
ib_dealloc_device(&dev->ibdev);
}
@ -320,8 +336,9 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev)
ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg);
xa_init_flags(&dev->qp_tbl, GFP_ATOMIC);
xa_init_flags(&dev->cq_tbl, GFP_ATOMIC);
xa_init_flags(&dev->qp_tbl, XA_FLAGS_LOCK_IRQ);
xa_init_flags(&dev->cq_tbl, XA_FLAGS_LOCK_IRQ);
xa_init_flags(&dev->srq_tbl, XA_FLAGS_LOCK_IRQ);
ionic_init_resids(dev);
@ -354,6 +371,8 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev)
goto err_admin;
ib_set_device_ops(&dev->ibdev, &ionic_dev_ops);
if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ))
ib_set_device_ops(&dev->ibdev, &ionic_srq_ops);
ionic_stats_init(dev);
@ -372,6 +391,7 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev)
ionic_destroy_resids(dev);
xa_destroy(&dev->qp_tbl);
xa_destroy(&dev->cq_tbl);
xa_destroy(&dev->srq_tbl);
ib_dealloc_device(&dev->ibdev);
return ERR_PTR(rc);

85
drivers/infiniband/hw/ionic/ionic_ibdev.h
View File

@ -36,6 +36,10 @@
#define IONIC_SQCMB_ORDER 5
#define IONIC_RQCMB_ORDER 0
#define IONIC_MAX_SRQ_SGES 2
#define IONIC_MAX_SRQ_LIMIT 0xffff
#define IONIC_MAX_SRQ_DEPTH 0xffff
#define IONIC_META_LAST ((void *)1ul)
#define IONIC_META_POSTED ((void *)2ul)
@ -88,6 +92,7 @@ struct ionic_ibdev {
struct xarray qp_tbl;
struct xarray cq_tbl;
struct xarray srq_tbl;
struct ionic_resid_bits inuse_dbid;
struct ionic_resid_bits inuse_pdid;
@ -241,6 +246,37 @@ struct ionic_rq_meta {
u64 wrid;
};
struct ionic_rq {
struct ionic_queue q;
u32 qid;
u32 cqid;
struct list_head cq_flush;
spinlock_t lock; /* for posting and polling */
phys_addr_t cmb_addr;
struct rdma_user_mmap_entry *mmap_cmb;
struct ionic_rq_meta *meta;
struct ionic_rq_meta *meta_head;
struct ib_umem *umem;
int spec;
int cmb_order;
u32 cmb_pgid;
u16 old_prod;
u8 cmb;
bool flush;
};
struct ionic_srq {
struct ib_srq ibsrq;
struct ionic_rq rq;
struct kref kref;
struct completion rel_comp;
u16 srq_limit;
u8 udma_idx;
};
struct ionic_qp {
struct ib_qp ibqp;
enum ib_qp_state state;
@ -248,18 +284,15 @@ struct ionic_qp {
u32 qpid;
u32 ahid;
u32 sq_cqid;
u32 rq_cqid;
u8 udma_idx;
u8 has_ah:1;
u8 has_sq:1;
u8 has_rq:1;
u8 sig_all:1;
struct list_head qp_list_counter;
struct list_head cq_poll_sq;
struct list_head cq_flush_sq;
struct list_head cq_flush_rq;
struct list_head ibkill_flush_ent;
spinlock_t sq_lock; /* for posting and polling */
@ -274,14 +307,7 @@ struct ionic_qp {
bool sq_flush;
bool sq_flush_rcvd;
spinlock_t rq_lock; /* for posting and polling */
struct ionic_queue rq;
struct ionic_rq_meta *rq_meta;
struct ionic_rq_meta *rq_meta_head;
int rq_spec;
u16 rq_old_prod;
u8 rq_cmb;
bool rq_flush;
struct ionic_rq rq;
struct kref qp_kref;
struct completion qp_rel_comp;
@ -295,16 +321,10 @@ struct ionic_qp {
struct ib_umem *sq_umem;
int rq_cmb_order;
u32 rq_cmb_pgid;
phys_addr_t rq_cmb_addr;
struct rdma_user_mmap_entry *mmap_rq_cmb;
struct ib_umem *rq_umem;
int dcqcn_profile;
struct ib_ud_header *hdr;
struct ionic_srq *srq;
};
struct ionic_ah {
@ -399,6 +419,11 @@ static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp)
return container_of(ibqp, struct ionic_qp, ibqp);
}
static inline struct ionic_srq *to_ionic_srq(struct ib_srq *ibsrq)
{
return container_of(ibsrq, struct ionic_srq, ibsrq);
}
static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah)
{
return container_of(ibah, struct ionic_ah, ibah);
@ -438,6 +463,19 @@ static inline void ionic_cq_complete(struct kref *kref)
complete(&cq->cq_rel_comp);
}
static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev,
enum ionic_lif_rdma_alloc_qid qtype)
{
return dev->lif_cfg.alloc_qid_cap & qtype;
}
static inline void ionic_srq_complete(struct kref *kref)
{
struct ionic_srq *srq = container_of(kref, struct ionic_srq, kref);
complete(&srq->rel_comp);
}
/* ionic_admin.c */
extern struct workqueue_struct *ionic_evt_workq;
void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);
@ -457,8 +495,11 @@ int ionic_create_cq_common(struct ionic_vcq *vcq,
struct ionic_ctx *ctx,
struct ib_udata *udata,
struct ionic_qdesc *req_cq,
__u32 *resp_cqid,
int udma_idx);
int ionic_post_create_cq_cmd(struct ionic_cq *cq,
struct ib_udata *udata,
__u32 *resp_cqid);
void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq);
void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);
void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);
void ionic_notify_flush_cq(struct ionic_cq *cq);
@ -498,6 +539,12 @@ int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
struct ib_qp_init_attr *init_attr);
int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr,
struct ib_udata *udata);
int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
/* ionic_datapath.c */
int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,

3
drivers/infiniband/hw/ionic/ionic_lif_cfg.c
View File

@ -75,6 +75,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg)
cfg->eq_count = lif->ionic->neqs_per_lif;
cfg->cq_count = le32_to_cpu(ident->rdma.cq_qtype.qid_count);
cfg->qp_count = le32_to_cpu(ident->rdma.sq_qtype.qid_count);
cfg->srq_count = le32_to_cpu(ident->rdma.srq_qtype.qid_count);
cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
cfg->aq_qtype = ident->rdma.aq_qtype.qtype;
@ -86,12 +87,14 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg)
cfg->udma_count = 2;
cfg->max_stride = ident->rdma.max_stride;
cfg->rcq_sign_bit = ident->rdma.rcq_sign_bit;
cfg->expdb_mask = ionic_get_expdb(lif);
cfg->sq_expdb =
!!(lif->qtype_info[IONIC_QTYPE_TXQ].features & IONIC_QIDENT_F_EXPDB);
cfg->rq_expdb =
!!(lif->qtype_info[IONIC_QTYPE_RXQ].features & IONIC_QIDENT_F_EXPDB);
cfg->alloc_qid_cap = ident->rdma.alloc_qid_cap;
}
struct net_device *ionic_lif_netdev(struct ionic_lif *lif)

3
drivers/infiniband/hw/ionic/ionic_lif_cfg.h
View File

@ -38,6 +38,7 @@ struct ionic_lif_cfg {
int eq_count;
int cq_count;
int qp_count;
int srq_count;
u16 stats_type;
u8 aq_qtype;
@ -57,6 +58,8 @@ struct ionic_lif_cfg {
bool sq_expdb;
bool rq_expdb;
u8 expdb_mask;
u8 rcq_sign_bit;
u8 alloc_qid_cap;
};
void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg);

31
drivers/infiniband/hw/irdma/ctrl.c
View File

@ -1689,11 +1689,9 @@ static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev,
* irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
* @qp: sc qp struct
* @info: fast mr info
* @post_sq: flag for cqp db to ring
*/
int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
struct irdma_fast_reg_stag_info *info,
bool post_sq)
struct irdma_fast_reg_stag_info *info)
{
u64 temp, hdr;
__le64 *wqe;
@ -1723,15 +1721,13 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
info->wr_id, wqe_idx,
&qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
(uintptr_t)info->va : info->fbo;
set_64bit_val(wqe, 0, temp);
set_64bit_val(wqe, 0, info->va);
temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI,
info->first_pm_pbl_index >> 16);
set_64bit_val(wqe, 8,
FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) |
FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa));
FIELD_PREP(IRDMAQPSQ_PBLADDR, info->reg_addr_pa >> IRDMA_HW_PAGE_SHIFT));
set_64bit_val(wqe, 16,
info->total_len |
FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index));
@ -1742,7 +1738,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) |
FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) |
FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) |
FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) |
IRDMAQPSQ_VABASEDTO |
FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
@ -1755,8 +1751,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8,
wqe, IRDMA_QP_WQE_MIN_SIZE, false);
if (post_sq)
irdma_uk_qp_post_wr(&qp->qp_uk);
irdma_uk_qp_post_wr(&qp->qp_uk);
return 0;
}
@ -2625,11 +2620,9 @@ static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,
* @qp: sc qp
* @info: dlush information
* @scratch: u64 saved to be used during cqp completion
* @post_sq: flag for cqp db to ring
*/
int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info, u64 scratch,
bool post_sq)
struct irdma_qp_flush_info *info, u64 scratch)
{
u64 temp = 0;
__le64 *wqe;
@ -2697,8 +2690,7 @@ int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8,
wqe, IRDMA_CQP_WQE_SIZE * 8, false);
if (post_sq)
irdma_sc_cqp_post_sq(cqp);
irdma_sc_cqp_post_sq(cqp);
return 0;
}
@ -5331,7 +5323,11 @@ int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
*/
static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp)
{
return IRDMA_RING_FULL_ERR(cqp->sq_ring);
u64 in_flight = cqp->requested_ops -
atomic64_read(&cqp->completed_ops);
return IRDMA_RING_FULL_ERR(cqp->sq_ring) ||
in_flight >= (cqp->sq_size - 1);
}
/**
@ -6114,8 +6110,7 @@ static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,
case IRDMA_OP_QP_FLUSH_WQES:
status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,
&pcmdinfo->in.u.qp_flush_wqes.info,
pcmdinfo->in.u.qp_flush_wqes.scratch,
pcmdinfo->post_sq);
pcmdinfo->in.u.qp_flush_wqes.scratch);
break;
case IRDMA_OP_GEN_AE:
status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,

16
drivers/infiniband/hw/irdma/hw.c
View File

@ -265,6 +265,7 @@ static void irdma_process_ae_def_cmpl(struct irdma_pci_f *rf,
irdma_sc_cqp_def_cmpl_ae_handler(&rf->sc_dev, info, false,
&scratch, &sw_def_info);
}
irdma_process_bh(&rf->sc_dev);
}
/**
@ -1252,12 +1253,21 @@ static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
}
msix_vec->ceq_id = ceq_id;
if (rf->sc_dev.privileged)
if (rf->sc_dev.privileged) {
rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id,
msix_vec->idx, true);
else
} else {
status = irdma_vchnl_req_ceq_vec_map(&rf->sc_dev, ceq_id,
msix_vec->idx);
if (!status)
return 0;
if (rf->msix_shared && !ceq_id)
irdma_destroy_irq(rf, msix_vec, rf);
else
irdma_destroy_irq(rf, msix_vec, iwceq);
}
return status;
}
@ -2742,7 +2752,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
hw_info = &cqp_request->info.in.u.qp_flush_wqes.info;
memcpy(hw_info, info, sizeof(*hw_info));
cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
cqp_info->post_sq = 1;
cqp_info->in.u.qp_flush_wqes.qp = qp;
cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request;
status = irdma_handle_cqp_op(rf, cqp_request);
@ -2789,7 +2798,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
hw_info = &new_req->info.in.u.qp_flush_wqes.info;
memcpy(hw_info, info, sizeof(*hw_info));
cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES;
cqp_info->post_sq = 1;
cqp_info->in.u.qp_flush_wqes.qp = qp;
cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)new_req;

9
drivers/infiniband/hw/irdma/icrdma_if.c
View File

@ -318,14 +318,15 @@ static void icrdma_remove(struct auxiliary_device *aux_dev)
container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev);
struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info;
struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev);
u8 rdma_ver = iwdev->rf->rdma_ver;
struct irdma_pci_f *rf = iwdev->rf;
u8 rdma_ver = rf->rdma_ver;
ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false);
irdma_ib_unregister_device(iwdev);
icrdma_deinit_interrupts(iwdev->rf, cdev_info);
mutex_destroy(&iwdev->rf->ah_tbl_lock);
icrdma_deinit_interrupts(rf, cdev_info);
mutex_destroy(&rf->ah_tbl_lock);
kfree(iwdev->rf);
kfree(rf);
pr_debug("INIT: Gen[%d] func[%d] device remove success\n",
rdma_ver, PCI_FUNC(cdev_info->pdev->devfn));

2
drivers/infiniband/hw/irdma/main.c
View File

@ -80,6 +80,8 @@ static int ig3rdma_vport_probe(struct auxiliary_device *aux_dev,
return -ENOMEM;
}
iwdev = ib_alloc_device(irdma_device, ibdev);
if (!iwdev)
return -ENOMEM;
/* Fill iwdev info */
iwdev->is_vport = true;
iwdev->rf = rf;

8
drivers/infiniband/hw/irdma/pble.c
View File

@ -330,7 +330,7 @@ static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc,
if (root->addr)
irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo);
kfree(lvl2->leafmem.va);
kvfree(lvl2->leafmem.va);
lvl2->leaf = NULL;
}
@ -358,7 +358,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
lvl2->leaf_cnt = total;
lvl2->leafmem.size = (sizeof(*leaf) * total);
lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL);
lvl2->leafmem.va = kvzalloc(lvl2->leafmem.size, GFP_KERNEL);
if (!lvl2->leafmem.va)
return -ENOMEM;
@ -367,7 +367,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo,
total << 3, &root->addr, &fpm_addr);
if (ret_code) {
kfree(lvl2->leafmem.va);
kvfree(lvl2->leafmem.va);
lvl2->leaf = NULL;
return -ENOMEM;
}
@ -482,7 +482,7 @@ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc,
status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl);
/* if level1_only, only go through it once */
if (!status || lvl)
if (!status || lvl == PBLE_LEVEL_1)
break;
}

3
drivers/infiniband/hw/irdma/protos.h
View File

@ -19,8 +19,7 @@ void irdma_sc_rt_init(struct irdma_sc_dev *dev);
void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp);
__le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch);
int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
struct irdma_fast_reg_stag_info *info,
bool post_sq);
struct irdma_fast_reg_stag_info *info);
/* HMC/FPM functions */
int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id);
/* stats misc */

7
drivers/infiniband/hw/irdma/type.h
View File

@ -1173,13 +1173,11 @@ struct irdma_reg_ns_stag_info {
struct irdma_fast_reg_stag_info {
u64 wr_id;
u64 reg_addr_pa;
u64 fbo;
void *va;
u64 va;
u64 total_len;
u32 page_size;
u32 chunk_size;
u32 first_pm_pbl_index;
enum irdma_addressing_type addr_type;
irdma_stag_index stag_idx;
u16 access_rights;
u32 pd_id;
@ -1383,8 +1381,7 @@ int irdma_sc_qp_create(struct irdma_sc_qp *qp,
int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq);
int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
struct irdma_qp_flush_info *info, u64 scratch,
bool post_sq);
struct irdma_qp_flush_info *info, u64 scratch);
int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info);
int irdma_sc_qp_modify(struct irdma_sc_qp *qp,
struct irdma_modify_qp_info *info, u64 scratch,

134
drivers/infiniband/hw/irdma/verbs.c
View File

@ -1576,9 +1576,17 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
issue_modify_qp = 1;
iwqp->suspend_pending = true;
break;
case IB_QPS_RESET:
/*
* A QP that was never moved to INIT has nothing to
* flush, so skip the hardware modify. Moving it to
* ERROR would make the following INIT fail.
*/
if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID)
break;
fallthrough;
case IB_QPS_SQE:
case IB_QPS_ERR:
case IB_QPS_RESET:
if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {
iwqp->ibqp_state = attr->qp_state;
spin_unlock_irqrestore(&iwqp->lock, flags);
@ -2909,42 +2917,6 @@ static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size)
return true;
}
/**
* irdma_check_mr_contiguous - check if MR is physically contiguous
* @palloc: pbl allocation struct
* @pg_size: page size
*/
static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc,
u32 pg_size)
{
struct irdma_pble_level2 *lvl2 = &palloc->level2;
struct irdma_pble_info *leaf = lvl2->leaf;
u64 *arr = NULL;
u64 *start_addr = NULL;
int i;
bool ret;
if (palloc->level == PBLE_LEVEL_1) {
arr = palloc->level1.addr;
ret = irdma_check_mem_contiguous(arr, palloc->total_cnt,
pg_size);
return ret;
}
start_addr = leaf->addr;
for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) {
arr = leaf->addr;
if ((*start_addr + ((u64)i * pg_size * PBLE_PER_PAGE)) != *arr)
return false;
ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size);
if (!ret)
return false;
}
return true;
}
/**
* irdma_setup_pbles - copy user pg address to pble's
* @rf: RDMA PCI function
@ -2981,9 +2953,6 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,
irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level);
if (lvl)
iwmr->pgaddrmem[0] = *pbl;
return 0;
}
@ -3405,24 +3374,33 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access,
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
struct ib_umem *region = iwmr->region;
u32 stag = 0;
u8 lvl;
int err;
lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
iwmr->access = access;
err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
if (err)
return err;
if (lvl) {
err = irdma_check_mr_contiguous(&iwpbl->pble_alloc,
if (ib_umem_is_contiguous(region)) {
/* HW needs a valid page size even without PBLEs, so use 4K. It
* behaves like a PBLE-backed region where the PBA must be page
* aligned. HW takes the page offset from the lower 12 VA bits.
*/
iwmr->page_size = SZ_4K;
iwmr->pgaddrmem[0] = ALIGN_DOWN(ib_umem_start_dma_addr(region),
iwmr->page_size);
if (err) {
irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
iwpbl->pbl_allocated = false;
}
} else {
iwmr->page_size = ib_umem_find_best_pgsz(region,
iwdev->rf->sc_dev.hw_attrs.page_size_cap,
iwmr->ibmr.iova);
if (unlikely(!iwmr->page_size))
return -EOPNOTSUPP;
iwmr->page_cnt = ib_umem_num_dma_blocks(region,
iwmr->page_size);
err = irdma_setup_pbles(iwdev->rf, iwmr,
PBLE_LEVEL_1 | PBLE_LEVEL_2);
if (err)
return err;
}
if (create_stag) {
@ -3458,10 +3436,8 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region,
struct ib_pd *pd, u64 virt,
enum irdma_memreg_type reg_type)
{
struct irdma_device *iwdev = to_iwdev(pd->device);
struct irdma_pbl *iwpbl;
struct irdma_mr *iwmr;
unsigned long pgsz_bitmap;
iwmr = kzalloc_obj(*iwmr);
if (!iwmr)
@ -3475,18 +3451,21 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region,
iwmr->ibmr.device = pd->device;
iwmr->ibmr.iova = virt;
iwmr->type = reg_type;
iwmr->len = region->length;
iwpbl->user_base = virt;
pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ?
iwdev->rf->sc_dev.hw_attrs.page_size_cap : SZ_4K;
/* Page size and count are calculated in irdma_reg_user_mr_type_mem()
* for MEM type MRs.
*/
if (reg_type == IRDMA_MEMREG_TYPE_MEM)
return iwmr;
iwmr->page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt);
iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K, virt);
if (unlikely(!iwmr->page_size)) {
kfree(iwmr);
return ERR_PTR(-EOPNOTSUPP);
}
iwmr->len = region->length;
iwpbl->user_base = virt;
iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size);
return iwmr;
@ -3825,7 +3804,6 @@ static int irdma_hwdereg_mr(struct ib_mr *ib_mr)
static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
u64 virt)
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
struct ib_pd *pd = iwmr->ibmr.pd;
struct ib_umem *region;
@ -3838,17 +3816,8 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
iwmr->region = region;
iwmr->ibmr.iova = virt;
iwmr->ibmr.pd = pd;
iwmr->page_size = ib_umem_find_best_pgsz(region,
iwdev->rf->sc_dev.hw_attrs.page_size_cap,
virt);
if (unlikely(!iwmr->page_size)) {
err = -EOPNOTSUPP;
goto err;
}
iwmr->len = region->length;
iwpbl->user_base = virt;
iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size);
err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false);
if (err)
@ -4276,16 +4245,27 @@ static int irdma_post_send(struct ib_qp *ibqp,
stag_info.stag_idx = reg_wr(ib_wr)->key >> 8;
stag_info.page_size = reg_wr(ib_wr)->mr->page_size;
stag_info.wr_id = ib_wr->wr_id;
stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED;
stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova;
stag_info.va = iwmr->ibmr.iova;
stag_info.total_len = iwmr->ibmr.length;
stag_info.reg_addr_pa = *palloc->level1.addr;
stag_info.first_pm_pbl_index = palloc->level1.idx;
stag_info.local_fence = true;
if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR)
stag_info.chunk_size = 1;
err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,
true);
if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) {
if (palloc->level == PBLE_LEVEL_1) {
stag_info.first_pm_pbl_index =
palloc->level1.idx;
stag_info.chunk_size = 1;
} else {
stag_info.first_pm_pbl_index =
palloc->level2.root.idx;
stag_info.chunk_size = 3;
}
} else {
stag_info.reg_addr_pa =
(palloc->level == PBLE_LEVEL_1) ?
*palloc->level1.addr :
*palloc->level2.leaf->addr;
}
err = irdma_sc_mr_fast_register(&iwqp->sc_qp,
&stag_info);
break;
}
default:

396
drivers/infiniband/hw/mana/cq.c
View File

@ -5,6 +5,58 @@
#include "mana_ib.h"
static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error)
{
switch (vendor_error) {
case VENDOR_ERR_OK:
return IB_WC_SUCCESS;
case VENDOR_ERR_RX_PKT_LEN:
case VENDOR_ERR_RX_MSG_LEN_OVFL:
return IB_WC_LOC_LEN_ERR;
case VENDOR_ERR_TX_GDMA_CORRUPTED_WQE:
case VENDOR_ERR_TX_PCIE_WQE:
case VENDOR_ERR_TX_PCIE_MSG:
case VENDOR_ERR_RX_MALFORMED_WQE:
case VENDOR_ERR_TX_GDMA_INVALID_STATE:
case VENDOR_ERR_TX_MISBEHAVING_CLIENT:
case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE:
case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD:
case VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED:
case VENDOR_ERR_TX_RDMA_WQE_LEN_ERR:
case VENDOR_ERR_TX_RDMA_MTU_ERR:
return IB_WC_LOC_QP_OP_ERR;
case VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION:
case VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE:
case VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR:
case VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION:
case VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE:
case VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR:
case VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE:
case VENDOR_ERR_RX_ATB_SGE_MISSCONFIG:
return IB_WC_LOC_PROT_ERR;
case VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT:
case VENDOR_ERR_RX_GFID:
return IB_WC_LOC_ACCESS_ERR;
case VENDOR_ERR_RX_MISBEHAVING_CLIENT:
case VENDOR_ERR_RX_CLIENT_ID:
case VENDOR_ERR_RX_PCIE:
case VENDOR_ERR_RX_NO_AVAIL_WQE:
case VENDOR_ERR_RX_ATB_WQE_MISCONFIG:
case VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT:
case VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE:
case VENDOR_ERR_TX_RDMA_INVALID_STATE:
case VENDOR_ERR_TX_RDMA_INVALID_NPT:
case VENDOR_ERR_TX_RDMA_INVALID_SGID:
case VENDOR_ERR_TX_RDMA_VFID_MISMATCH:
return IB_WC_FATAL_ERR;
case VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE:
case VENDOR_ERR_SW_FLUSHED:
return IB_WC_WR_FLUSH_ERR;
default:
return IB_WC_GENERAL_ERR;
}
}
int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct uverbs_attr_bundle *attrs)
{
@ -40,7 +92,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
cq->cqe = attr->cqe;
err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE,
&cq->queue);
&cq->queue, true);
if (err) {
ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err);
return err;
@ -64,6 +116,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
doorbell = mdev->gdma_dev->doorbell;
}
ibcq->cqe = cq->cqe;
cq->poll_credit = (cq->cqe << (GDMA_CQE_OWNER_BITS - 1)) - 1;
if (is_rnic_cq) {
err = mana_ib_gd_create_cq(mdev, cq, doorbell);
if (err) {
@ -86,8 +141,8 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
}
spin_lock_init(&cq->cq_lock);
INIT_LIST_HEAD(&cq->list_send_qp);
INIT_LIST_HEAD(&cq->list_recv_qp);
INIT_LIST_HEAD(&cq->send_err_qp_list);
INIT_LIST_HEAD(&cq->recv_err_qp_list);
return 0;
@ -174,171 +229,290 @@ void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
gc->cq_table[cq->queue.id] = NULL;
}
static inline bool gdma_cq_idx_produced(struct gdma_queue *gdma_cq, uint32_t idx)
{
struct gdma_mem_info *gmi = &gdma_cq->mem_info;
u32 num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE;
u32 expected_bits = (idx / num_cqe) & GDMA_CQE_OWNER_MASK;
u32 offset = (idx % num_cqe) * GDMA_CQE_SIZE;
struct gdma_cqe *cqe;
if (gmi->nr_pages)
cqe = gmi->pages_va[offset / PAGE_SIZE] +
(offset & (PAGE_SIZE - 1));
else
cqe = gdma_cq->queue_mem_ptr + offset;
return cqe->cqe_info.owner_bits == expected_bits;
}
static inline void mana_ib_cq_doorbell(struct mana_ib_cq *cq, uint8_t arm)
{
struct mana_ib_dev *mdev = container_of(cq->ibcq.device, struct mana_ib_dev, ib_dev);
struct gdma_queue *gdma_cq = cq->queue.kmem;
u32 num_cqe, max_credit, idx;
num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE;
max_credit = num_cqe << (GDMA_CQE_OWNER_BITS - 1);
idx = gdma_cq->head;
if (cq->poll_credit >= max_credit) {
if (gdma_cq_idx_produced(gdma_cq, idx + cq->poll_credit - max_credit))
cq->poll_credit++;
else
return;
} else {
/* Set index of already polled CQE for unarm */
cq->poll_credit = max_credit - (arm ? 0 : 1);
}
idx += (cq->poll_credit - max_credit);
idx %= (num_cqe << GDMA_CQE_OWNER_BITS);
mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), gdma_cq, idx, arm, 0);
}
int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
{
struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
struct gdma_queue *gdma_cq = cq->queue.kmem;
unsigned long irq_flags;
if (!gdma_cq)
return -EINVAL;
mana_gd_ring_cq(gdma_cq, SET_ARM_BIT);
spin_lock_irqsave(&cq->cq_lock, irq_flags);
mana_ib_cq_doorbell(cq, SET_ARM_BIT);
spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
return 0;
}
static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
struct mana_cq_poll {
struct ib_wc *wc;
int budget;
int produced;
};
static struct ib_wc *mana_fill_wc(struct mana_ib_qp *qp,
struct mana_cq_poll *poll,
const struct shadow_wqe_header *wqe,
enum ib_wc_opcode opcode, u32 vendor_error)
{
struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
struct ud_sq_shadow_wqe *shadow_wqe;
struct ib_wc *wc = &poll->wc[poll->produced++];
shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq);
if (!shadow_wqe)
return;
memset(wc, 0, sizeof(*wc));
wc->wr_id = wqe->wr_id;
wc->status = vendor_error_to_wc_error(vendor_error);
wc->opcode = opcode;
wc->vendor_err = vendor_error;
wc->qp = &qp->ibqp;
shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error;
wq->tail += shadow_wqe->header.posted_wqe_size;
shadow_queue_advance_next_to_complete(&qp->shadow_sq);
return wc;
}
static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe)
static void mana_complete_send(struct mana_ib_qp *qp,
struct mana_cq_poll *poll, u32 vendor_error)
{
struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
struct ud_rq_shadow_wqe *shadow_wqe;
struct shadow_queue *shadow = &qp->shadow_sq;
struct shadow_wqe_header *wqe = shadow_queue_get_next_to_consume(shadow);
struct gdma_queue *queue;
shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq);
if (!shadow_wqe)
if (!wqe)
return;
shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len;
shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn;
shadow_wqe->header.error_code = IB_WC_SUCCESS;
if (vendor_error || !(wqe->flags & MANA_WQ_NO_SIGNAL_WC))
mana_fill_wc(qp, poll, wqe, wqe->send_opcode, vendor_error);
wq->tail += shadow_wqe->header.posted_wqe_size;
shadow_queue_advance_next_to_complete(&qp->shadow_rq);
queue = mana_qp_get_sq(qp)->kmem;
queue->tail += wqe->wqe_size_in_bu;
shadow_queue_advance_consumer(shadow);
}
static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe)
static void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct mana_rdma_cqe *rdma_cqe,
struct mana_cq_poll *poll)
{
u32 offset = rdma_cqe->ud_send.tx_wqe_offset & MANA_WQE_OFFSET_MASK;
struct shadow_queue *shadow = &qp->shadow_sq;
struct shadow_wqe_header *wqe;
u64 idx = shadow->cons_idx;
u32 to_complete = 0;
u64 prod_idx;
/* Pair with posting's release of the initialized shadow entries. */
prod_idx = smp_load_acquire(&shadow->prod_idx);
/* Find the target before retiring any entries: the CQE may be stale. */
for (; idx != prod_idx; idx++) {
wqe = shadow_queue_get_element(shadow, idx);
to_complete++;
if (wqe->wqe_offset_or_psn == offset)
break;
if (!(wqe->flags & MANA_WQ_NO_SIGNAL_WC))
return;
}
if (idx == prod_idx)
return;
for (; to_complete; to_complete--)
mana_complete_send(qp, poll, VENDOR_ERR_OK);
}
static void handle_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe,
struct mana_cq_poll *poll)
{
struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
u32 offset = rdma_cqe->ud_recv.rx_wqe_offset / GDMA_WQE_BU_SIZE;
struct mana_ib_queue *rq = mana_qp_get_rq(qp);
struct gdma_queue *wq = rq->kmem;
struct shadow_wqe_header *wqe;
struct ib_wc *wc;
wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq);
if (!wqe || wqe->wqe_offset_or_psn != (offset & MANA_WQE_OFFSET_MASK))
return;
wc = mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_OK);
switch (rdma_cqe->cqe_type) {
case CQE_TYPE_UD_SEND_IMM:
wc->ex.imm_data = cpu_to_be32(rdma_cqe->ud_recv.imm_data);
wc->wc_flags |= IB_WC_WITH_IMM;
fallthrough;
case CQE_TYPE_UD_SEND:
wc->byte_len = rdma_cqe->ud_recv.msg_len;
wc->src_qp = rdma_cqe->ud_recv.src_qpn;
wc->wc_flags |= IB_WC_GRH;
break;
default:
break;
}
wq->tail += wqe->wqe_size_in_bu;
shadow_queue_advance_consumer(&qp->shadow_rq);
}
static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev,
struct mana_cq_poll *poll)
{
struct gdma_comp *cqe = &cq->pending_cqe;
struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data;
struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq);
if (!qp)
return true;
switch (rdma_cqe->cqe_type) {
case CQE_TYPE_UD_SEND:
if (cqe->is_sq) {
handle_ud_sq_cqe(qp, rdma_cqe, poll);
break;
}
fallthrough;
case CQE_TYPE_UD_SEND_IMM:
handle_rq_cqe(qp, cqe, poll);
break;
default:
ibdev_warn_ratelimited(qp->ibqp.device, "Unexpected CQE type %u\n",
rdma_cqe->cqe_type);
break;
}
mana_put_qp_ref(qp);
return true;
}
static void mana_flush_completions(struct mana_ib_cq *cq, struct mana_cq_poll *poll)
{
struct shadow_wqe_header *wqe;
struct mana_ib_qp *qp;
if (poll->produced >= poll->budget)
return;
if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) {
if (cqe->is_sq)
handle_ud_sq_cqe(qp, cqe);
else
handle_ud_rq_cqe(qp, cqe);
list_for_each_entry(qp, &cq->send_err_qp_list, send_err_node) {
while (poll->produced < poll->budget &&
shadow_queue_get_next_to_consume(&qp->shadow_sq))
mana_complete_send(qp, poll, VENDOR_ERR_SW_FLUSHED);
if (poll->produced == poll->budget)
return;
}
mana_put_qp_ref(qp);
}
static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc,
const struct shadow_wqe_header *shadow_wqe)
{
const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe;
wc->wr_id = shadow_wqe->wr_id;
wc->status = shadow_wqe->error_code;
wc->opcode = shadow_wqe->opcode;
wc->vendor_err = shadow_wqe->error_code;
wc->wc_flags = 0;
wc->qp = &qp->ibqp;
wc->pkey_index = 0;
if (shadow_wqe->opcode == IB_WC_RECV) {
wc->byte_len = ud_wqe->byte_len;
wc->src_qp = ud_wqe->src_qpn;
wc->wc_flags |= IB_WC_GRH;
}
}
static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc)
{
struct shadow_wqe_header *shadow_wqe;
struct mana_ib_qp *qp;
int wc_index = 0;
/* process send shadow queue completions */
list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) {
while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq))
!= NULL) {
if (wc_index >= nwc)
goto out;
fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
shadow_queue_advance_consumer(&qp->shadow_sq);
wc_index++;
}
}
/* process recv shadow queue completions */
list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) {
while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))
!= NULL) {
if (wc_index >= nwc)
goto out;
fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe);
list_for_each_entry(qp, &cq->recv_err_qp_list, recv_err_node) {
while (poll->produced < poll->budget &&
(wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))) {
mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_SW_FLUSHED);
shadow_queue_advance_consumer(&qp->shadow_rq);
wc_index++;
}
if (poll->produced == poll->budget)
return;
}
out:
return wc_index;
}
void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)
static void mana_drain_gsi_sq(struct mana_ib_qp *qp)
{
struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false);
struct ud_sq_shadow_wqe *shadow_wqe;
struct mana_ib_cq *cq;
struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
unsigned long flags;
if (!qp)
return;
cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
spin_lock_irqsave(&cq->cq_lock, flags);
while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq))
!= NULL) {
shadow_wqe->header.error_code = IB_WC_GENERAL_ERR;
shadow_queue_advance_next_to_complete(&qp->shadow_sq);
}
if (list_empty(&qp->send_err_node))
list_add_tail(&qp->send_err_node, &cq->send_err_qp_list);
spin_unlock_irqrestore(&cq->cq_lock, flags);
if (cq->ibcq.comp_handler)
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
mana_put_qp_ref(qp);
void mana_drain_gsi_sqs(struct mana_ib_dev *mdev)
{
struct mana_ib_qp *qp;
u32 port;
/* One GSI QP per port, indexed in the QP table by (port << 24 | MANA_GSI_QPN) */
for (port = 1; port <= mdev->ib_dev.phys_port_cnt; port++) {
qp = mana_get_qp_ref(mdev, (port << 24) | MANA_GSI_QPN, false);
if (!qp)
continue;
mana_drain_gsi_sq(qp);
mana_put_qp_ref(qp);
}
}
int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
{
struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq);
struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev);
struct mana_cq_poll poll = { .wc = wc, .budget = num_entries, .produced = 0 };
struct gdma_queue *queue = cq->queue.kmem;
struct gdma_comp gdma_cqe;
unsigned long flags;
int num_polled = 0;
int comp_read, i;
bool consumed;
int comp_read;
if (!queue)
return -EINVAL;
spin_lock_irqsave(&cq->cq_lock, flags);
for (i = 0; i < num_entries; i++) {
comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1);
if (comp_read < 1)
break;
mana_handle_cqe(mdev, &gdma_cqe);
while (poll.produced < poll.budget) {
if (!cq->has_pending_cqe) {
comp_read = mana_gd_poll_cq(queue, &cq->pending_cqe, 1);
if (comp_read < 0) {
if (!poll.produced)
poll.produced = comp_read;
goto out;
}
if (!comp_read)
break;
cq->poll_credit--;
if (!cq->poll_credit)
mana_ib_cq_doorbell(cq, 0);
}
consumed = mana_handle_cqe(cq, mdev, &poll);
cq->has_pending_cqe = !consumed;
}
num_polled = mana_process_completions(cq, num_entries, wc);
mana_flush_completions(cq, &poll);
out:
spin_unlock_irqrestore(&cq->cq_lock, flags);
return num_polled;
return poll.produced;
}

3
drivers/infiniband/hw/mana/device.c
View File

@ -103,8 +103,7 @@ static int mana_ib_netdev_event(struct notifier_block *this,
ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
/* mana_get_primary_netdev() returns ndev with refcount held */
if (ndev)
netdev_put(ndev, &dev->dev_tracker);
netdev_put(ndev, &dev->dev_tracker);
return NOTIFY_OK;
default:

43
drivers/infiniband/hw/mana/main.c
View File

@ -247,6 +247,9 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext,
ucontext->doorbell = doorbell_page;
ucmd_resp.comp_mask = MANA_IB_UCNTX_ALLOC_PDN_SUPPORT;
ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_EXT_SUPPORT;
if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT)
ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_SQ_POW2_SUPPORT;
ret = ib_respond_udata(udata, ucmd_resp);
if (ret)
return ret;
@ -292,7 +295,7 @@ int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_qu
}
int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
struct mana_ib_queue *queue)
struct mana_ib_queue *queue, bool is_cq)
{
struct ib_umem *umem;
int err;
@ -301,7 +304,12 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
queue->id = INVALID_QUEUE_ID;
queue->gdma_region = GDMA_INVALID_DMA_REGION;
umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE);
if (is_cq)
umem = ib_umem_get_cq_buf_or_va(&mdev->ib_dev, NULL, addr,
size, IB_ACCESS_LOCAL_WRITE);
else
umem = ib_umem_get_va(&mdev->ib_dev, addr, size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem);
return PTR_ERR(umem);
@ -592,7 +600,11 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
immutable->gid_tbl_len = attr.gid_tbl_len;
if (mana_ib_is_rnic(dev)) {
if (port_num == 1) {
bool port_supports_cm = (port_num == 1 ||
(dev->adapter_caps.feature_flags &
MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT));
if (port_supports_cm) {
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
} else {
@ -670,10 +682,14 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width);
props->pkey_tbl_len = 1;
if (mana_ib_is_rnic(dev)) {
bool port_supports_cm = (port == 1 ||
(dev->adapter_caps.feature_flags &
MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT));
props->gid_tbl_len = 16;
props->ip_gids = true;
props->max_msg_sz = SZ_16M;
if (port == 1)
if (port_supports_cm)
props->port_cap_flags = IB_PORT_CM_SUP;
}
@ -1059,6 +1075,9 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
req.max_recv_sge = attr->cap.max_recv_sge;
req.flags = flags;
if (flags & MANA_RC_FLAG_FIXED_SIZE_WQE)
req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu;
err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
if (err)
return err;
@ -1134,6 +1153,7 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
struct gdma_context *gc = mdev_to_gc(mdev);
struct mana_rnic_create_udqp_resp resp = {};
struct mana_rnic_create_udqp_req req = {};
struct net_device *ndev;
int err, i;
mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(resp));
@ -1150,6 +1170,21 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
req.max_send_sge = attr->cap.max_send_sge;
req.max_recv_sge = attr->cap.max_recv_sge;
req.qp_type = type;
/* For GSI QPs, pass the vNIC MAC so the SoC can associate the QP with
* the correct port, transition to INIT, and allocate a per-port QPN.
* MAC must be in reversed byte order.
*/
if (type == IB_QPT_GSI &&
(mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)) {
ndev = mana_ib_get_netdev(&mdev->ib_dev, attr->port_num);
if (ndev) {
copy_in_reverse(req.mac, ndev->dev_addr, ETH_ALEN);
req.flags = MANA_UD_QP_FLAG_CREATE_IN_INIT;
req.hdr.req.msg_version = GDMA_MESSAGE_V2;
}
}
err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
if (err)
return err;

134
drivers/infiniband/hw/mana/mana_ib.h
View File

@ -24,8 +24,12 @@
/* MANA doesn't have any limit for MR size */
#define MANA_IB_MAX_MR_SIZE U64_MAX
/* Send queue ID mask */
#define MANA_SENDQ_MASK BIT(31)
/*
* Send queue ID mask. Queue IDs are 2-bit aligned (see MANA_QID_SUBTYPE_MASK),
* so bit 0 is always free to tag send queues in the lookup table. This keeps
* the whole top byte available to index per-port GSI QPs by (port << 24).
*/
#define MANA_SENDQ_MASK BIT(0)
/* Queue ID encodes type in the lower 2 bits */
#define MANA_QID_SUBTYPE_MASK 0x3
@ -162,17 +166,20 @@ struct mana_ib_cq {
struct mana_ib_queue queue;
/* protects CQ polling */
spinlock_t cq_lock;
struct list_head list_send_qp;
struct list_head list_recv_qp;
struct list_head send_err_qp_list;
struct list_head recv_err_qp_list;
struct gdma_comp pending_cqe;
bool has_pending_cqe;
int cqe;
u32 comp_vector;
u32 poll_credit;
mana_handle_t cq_handle;
};
enum mana_rc_queue_type {
MANA_RC_SEND_QUEUE_REQUESTER = 0,
MANA_RC_SEND_QUEUE_RESPONDER,
MANA_RC_SEND_QUEUE_FMR,
MANA_RC_SEND_QUEUE_MMQ,
MANA_RC_RECV_QUEUE_REQUESTER,
MANA_RC_RECV_QUEUE_RESPONDER,
MANA_RC_QUEUE_TYPE_MAX,
@ -180,6 +187,7 @@ enum mana_rc_queue_type {
struct mana_ib_rc_qp {
struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
u32 wqe_size_in_bu;
};
enum mana_uc_queue_type {
@ -201,7 +209,6 @@ enum mana_ud_queue_type {
struct mana_ib_ud_qp {
struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX];
u32 sq_psn;
};
struct mana_ib_qp {
@ -217,9 +224,14 @@ struct mana_ib_qp {
/* The port on the IB device, starting with 1 */
u32 port;
u32 sq_psn;
bool sq_sig_all;
struct list_head cq_send_list;
struct list_head cq_recv_list;
/* Serializes QP modification and error-list transitions. */
struct mutex modify_lock;
struct list_head send_err_node;
struct list_head recv_err_node;
struct shadow_queue shadow_rq;
struct shadow_queue shadow_sq;
@ -262,6 +274,9 @@ enum mana_ib_adapter_features {
MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7),
MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT = BIT(14),
MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT = BIT(15),
};
struct mana_ib_query_adapter_caps_resp {
@ -372,7 +387,9 @@ struct mana_rnic_destroy_cq_resp {
}; /* HW Data */
enum mana_rnic_create_rc_flags {
MANA_RC_FLAG_NO_FMR = 2,
MANA_RC_FLAG_NO_MMQ = BIT(1),
MANA_RC_FLAG_FIXED_SIZE_WQE = BIT(3),
MANA_RC_FLAG_MSN_IN_WQE = BIT(4),
};
struct mana_rnic_create_qp_req {
@ -389,7 +406,8 @@ struct mana_rnic_create_qp_req {
u32 max_recv_wr;
u32 max_send_sge;
u32 max_recv_sge;
u32 reserved;
u8 wqe_size_in_bu;
u8 reserved[3];
}; /* HW Data */
struct mana_rnic_create_qp_resp {
@ -445,8 +463,14 @@ struct mana_rnic_create_udqp_req {
u32 max_recv_wr;
u32 max_send_sge;
u32 max_recv_sge;
u8 mac[ETH_ALEN]; /* V2: port MAC for multi-port GSI */
u16 flags; /* V2: MANA_UD_QP_FLAG_* */
}; /* HW Data */
enum mana_ud_qp_flags {
MANA_UD_QP_FLAG_CREATE_IN_INIT = BIT(0),
};
struct mana_rnic_create_udqp_resp {
struct gdma_resp_hdr hdr;
mana_handle_t qp_handle;
@ -508,7 +532,16 @@ struct mana_rnic_set_qp_state_resp {
enum WQE_OPCODE_TYPES {
WQE_TYPE_UD_SEND = 0,
WQE_TYPE_RC_SEND = 2,
WQE_TYPE_RC_SEND_IMM = 3,
WQE_TYPE_RC_SEND_INV = 4,
WQE_TYPE_WRITE = 5,
WQE_TYPE_WRITE_IMM = 6,
WQE_TYPE_READ = 7,
WQE_TYPE_UD_RECV = 8,
WQE_TYPE_RC_RECV = 9,
WQE_TYPE_REG_MR = 10,
WQE_TYPE_LOCAL_INV = 12,
}; /* HW DATA */
struct rdma_send_oob {
@ -527,7 +560,83 @@ struct rdma_send_oob {
u32 reserved1;
u32 reserved2;
} ud_send;
union {
u32 immediate;
u32 invalidate_key;
} rc_send;
struct {
u32 address_hi;
u32 address_low;
u32 rkey;
u32 dma_len;
} rdma;
struct {
u32 mkey;
} mm;
};
union {
u32 immediate_ext;
struct {
u16 rsn;
u16 reserved;
} read;
};
u32 fsn : 24;
u32 reserved2 : 8;
}; /* HW DATA */
struct rdma_recv_oob {
u32 psn_start : 24;
u32 reserved1 : 8;
u32 msn : 24;
u32 reserved2 : 8;
}; /* HW DATA */
enum mana_ib_error_code {
VENDOR_ERR_OK = 0x0,
VENDOR_ERR_RX_PKT_LEN = 0x05,
VENDOR_ERR_RX_MSG_LEN_OVFL = 0x102,
VENDOR_ERR_RX_MISBEHAVING_CLIENT = 0x108,
VENDOR_ERR_RX_MALFORMED_WQE = 0x109,
VENDOR_ERR_RX_CLIENT_ID = 0x10a,
VENDOR_ERR_RX_GFID = 0x10b,
VENDOR_ERR_RX_PCIE = 0x10c,
VENDOR_ERR_RX_NO_AVAIL_WQE = 0x111,
VENDOR_ERR_RX_ATB_SGE_MISSCONFIG = 0x143,
VENDOR_ERR_RX_ATB_WQE_MISCONFIG = 0x145,
VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT = 0x183,
VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT = 0x185,
VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE = 0x1c3,
VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE = 0x1c5,
VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE = 0x1c7,
VENDOR_ERR_TX_GDMA_CORRUPTED_WQE = 0x201,
VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION = 0x202,
VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE = 0x203,
VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR = 0x204,
VENDOR_ERR_TX_PCIE_WQE = 0x205,
VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION = 0x206,
VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE = 0x207,
VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR = 0x208,
VENDOR_ERR_TX_PCIE_MSG = 0x209,
VENDOR_ERR_TX_GDMA_INVALID_STATE = 0x20a,
VENDOR_ERR_TX_MISBEHAVING_CLIENT = 0x20b,
VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE = 0x210,
VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD = 0x211,
VENDOR_ERR_TX_RDMA_INVALID_STATE = 0x212,
VENDOR_ERR_TX_RDMA_INVALID_NPT = 0x213,
VENDOR_ERR_TX_RDMA_INVALID_SGID = 0x214,
VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED = 0x215,
VENDOR_ERR_TX_RDMA_WQE_LEN_ERR = 0x216,
VENDOR_ERR_TX_RDMA_MTU_ERR = 0x217,
VENDOR_ERR_TX_RDMA_VFID_MISMATCH = 0x218,
VENDOR_ERR_HW_MAX = 0x3ff,
/* SW vendor errors */
VENDOR_ERR_SW_FLUSHED = 0xfff,
};
enum mana_ib_cqe_type {
CQE_TYPE_UD_SEND = 1,
CQE_TYPE_UD_SEND_IMM = 2,
}; /* HW DATA */
struct mana_rdma_cqe {
@ -538,8 +647,7 @@ struct mana_rdma_cqe {
};
struct {
u32 cqe_type : 8;
u32 vendor_error : 9;
u32 reserved1 : 15;
u32 reserved1 : 24;
u32 sge_offset : 5;
u32 tx_wqe_offset : 27;
} ud_send;
@ -714,7 +822,7 @@ int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev,
int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type,
struct mana_ib_queue *queue);
int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size,
struct mana_ib_queue *queue);
struct mana_ib_queue *queue, bool is_cq);
void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue);
struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,

150
drivers/infiniband/hw/mana/qp.c
View File

@ -340,7 +340,8 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n",
ucmd.sq_buf_addr, ucmd.port);
err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq);
err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq,
false);
if (err) {
ibdev_dbg(&mdev->ib_dev,
"Failed to create queue for create qp-raw, err %d\n", err);
@ -422,6 +423,11 @@ static u32 mana_ib_wqe_size(u32 sge, u32 oob_size)
return ALIGN(wqe_size, GDMA_WQE_BU_SIZE);
}
static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size)
{
return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size));
}
static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type)
{
u32 queue_size;
@ -509,8 +515,20 @@ static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
if (err)
goto err_remove_sq;
/* GSI QPs are additionally indexed by (port << 24 | MANA_GSI_QPN) so the
* per-port GSI SQ drain can find each of them by iterating ports.
*/
if (qp->ibqp.qp_type == IB_QPT_GSI) {
err = mana_table_store_qp_qid(mdev, qp,
(qp->port << 24) | MANA_GSI_QPN, false);
if (err)
goto err_remove_rq;
}
return 0;
err_remove_rq:
mana_table_remove_qp_qid(mdev, rq->id, false);
err_remove_sq:
mana_table_remove_qp_qid(mdev, sq->id, true);
mana_table_drain_qp_ref(qp);
@ -529,6 +547,8 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp
mana_table_remove_qp_qid(mdev, sq->id, true);
mana_table_remove_qp_qid(mdev, rq->id, false);
if (qp->ibqp.qp_type == IB_QPT_GSI)
mana_table_remove_qp_qid(mdev, (qp->port << 24) | MANA_GSI_QPN, false);
mana_table_drain_qp_ref(qp);
}
@ -549,27 +569,42 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext);
doorbell = mana_ucontext->doorbell;
flags = MANA_RC_FLAG_NO_FMR;
err = ib_copy_validate_udata_in(udata, ucmd, queue_size);
flags = MANA_RC_FLAG_NO_MMQ;
err = ib_copy_validate_udata_in_cm(udata, ucmd, queue_size,
MANA_IB_RC_QP_FIXED_WQE | MANA_IB_RC_MMQ_CREATE);
if (err)
return err;
for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
/* skip FMR for user-level RC QPs */
if (i == MANA_RC_SEND_QUEUE_FMR) {
qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
if (i == MANA_RC_SEND_QUEUE_MMQ) {
if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE) {
flags &= ~MANA_RC_FLAG_NO_MMQ;
err = mana_ib_create_queue(mdev, ucmd.mmq_buf, ucmd.mmq_size,
&qp->rc_qp.queues[i], false);
if (err)
goto destroy_queues;
} else {
qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
}
continue;
}
err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],
&qp->rc_qp.queues[i]);
if (err) {
ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err);
&qp->rc_qp.queues[i], false);
if (err)
goto destroy_queues;
}
j++;
}
if (ucmd.comp_mask & MANA_IB_RC_QP_FIXED_WQE) {
u32 wqe_size = mana_ib_fixed_wqe_size(attr->cap.max_send_sge,
INLINE_OOB_EXTRA_LARGE_SIZE);
flags |= MANA_RC_FLAG_FIXED_SIZE_WQE;
if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT)
flags |= MANA_RC_FLAG_MSN_IN_WQE;
qp->rc_qp.wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE;
}
err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
if (err) {
ibdev_err(&mdev->ib_dev, "Failed to create rc qp %d\n", err);
@ -580,8 +615,10 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
if (udata) {
for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
if (i == MANA_RC_SEND_QUEUE_FMR)
if (i == MANA_RC_SEND_QUEUE_MMQ) {
resp.mmq_id = qp->rc_qp.queues[i].id;
continue;
}
resp.queue_id[j] = qp->rc_qp.queues[i].id;
j++;
}
@ -627,7 +664,7 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; ++i) {
err = mana_ib_create_queue(mdev, ucmd.queue_buf[i], ucmd.queue_size[i],
&qp->uc_qp.queues[i]);
&qp->uc_qp.queues[i], false);
if (err)
goto destroy_queues;
}
@ -658,33 +695,23 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
return err;
}
static void mana_add_qp_to_cqs(struct mana_ib_qp *qp)
static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp, bool reset)
{
struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
unsigned long flags;
spin_lock_irqsave(&send_cq->cq_lock, flags);
list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
list_del_init(&qp->send_err_node);
/* Keep shadow reset serialized with hardware polling and SW flushing. */
if (reset)
reset_shadow_queue(&qp->shadow_sq);
spin_unlock_irqrestore(&send_cq->cq_lock, flags);
spin_lock_irqsave(&recv_cq->cq_lock, flags);
list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
}
static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp)
{
struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
unsigned long flags;
spin_lock_irqsave(&send_cq->cq_lock, flags);
list_del(&qp->cq_send_list);
spin_unlock_irqrestore(&send_cq->cq_lock, flags);
spin_lock_irqsave(&recv_cq->cq_lock, flags);
list_del(&qp->cq_recv_list);
list_del_init(&qp->recv_err_node);
if (reset)
reset_shadow_queue(&qp->shadow_rq);
spin_unlock_irqrestore(&recv_cq->cq_lock, flags);
}
@ -712,13 +739,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
doorbell = mdev->gdma_dev->doorbell;
err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr,
sizeof(struct ud_rq_shadow_wqe));
sizeof(struct shadow_wqe_header));
if (err) {
ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err);
goto destroy_queues;
}
err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr,
sizeof(struct ud_sq_shadow_wqe));
sizeof(struct shadow_wqe_header));
if (err) {
ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err);
goto destroy_shadow_queues;
@ -729,7 +756,8 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
ibdev_err(&mdev->ib_dev, "Failed to create ud qp %d\n", err);
goto destroy_shadow_queues;
}
qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
qp->ibqp.qp_num = (qp->ibqp.qp_type == IB_QPT_GSI) ?
MANA_GSI_QPN : qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id;
qp->port = attr->port_num;
for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i)
@ -739,8 +767,6 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
if (err)
goto destroy_qp;
mana_add_qp_to_cqs(qp);
return 0;
destroy_qp:
@ -757,6 +783,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
struct ib_udata *udata)
{
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
qp->sq_sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR;
mutex_init(&qp->modify_lock);
INIT_LIST_HEAD(&qp->send_err_node);
INIT_LIST_HEAD(&qp->recv_err_node);
switch (attr->qp_type) {
case IB_QPT_RAW_PACKET:
/* When rwq_ind_tbl is used, it's for creating WQs for RSS */
@ -854,19 +887,56 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
return 0;
}
static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
if (udata)
return;
if (attr_mask & IB_QP_STATE) {
switch (attr->qp_state) {
case IB_QPS_RESET:
mana_remove_qp_from_cqs(qp, true);
break;
default:
break;
}
}
if (attr_mask & IB_QP_SQ_PSN)
qp->sq_psn = attr->sq_psn;
}
int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
int ret;
mutex_lock(&qp->modify_lock);
switch (ibqp->qp_type) {
case IB_QPT_RC:
case IB_QPT_UC:
case IB_QPT_UD:
case IB_QPT_GSI:
return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
ret = mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
if (ret)
goto out_unlock;
break;
default:
ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type);
return -EOPNOTSUPP;
ret = -EOPNOTSUPP;
goto out_unlock;
}
mana_ib_modify_qp_state(ibqp, attr, attr_mask, udata);
out_unlock:
mutex_unlock(&qp->modify_lock);
return ret;
}
static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
@ -1003,8 +1073,8 @@ static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
if (err)
return err;
mana_remove_qp_from_cqs(qp);
mana_table_remove_qp(mdev, qp);
mana_remove_qp_from_cqs(qp, false);
destroy_shadow_queue(&qp->shadow_rq);
destroy_shadow_queue(&qp->shadow_sq);

61
drivers/infiniband/hw/mana/shadow_queue.h
View File

@ -6,21 +6,19 @@
#ifndef _MANA_SHADOW_QUEUE_H_
#define _MANA_SHADOW_QUEUE_H_
#include <linux/build_bug.h>
#define MANA_WQ_FENCE_WC BIT(0)
#define MANA_WQ_NO_SIGNAL_WC BIT(1)
#define MANA_WQE_OFFSET_MASK GENMASK(23, 0)
struct shadow_wqe_header {
u16 opcode;
u16 error_code;
u32 posted_wqe_size;
u64 wr_id;
};
struct ud_rq_shadow_wqe {
struct shadow_wqe_header header;
u32 byte_len;
u32 src_qpn;
};
struct ud_sq_shadow_wqe {
struct shadow_wqe_header header;
u64 wqe_offset_or_psn : 24;
u64 wqe_size_in_bu : 8;
u64 fsn : 24;
u64 send_opcode : 4;
u64 flags : 2;
};
struct shadow_queue {
@ -28,8 +26,6 @@ struct shadow_queue {
u64 prod_idx;
/* Unmasked consumer index, Incremented on cq polling */
u64 cons_idx;
/* Unmasked index of next-to-complete (from HW) shadow WQE */
u64 next_to_complete_idx;
/* queue size in wqes */
u32 length;
/* distance between elements in bytes */
@ -50,6 +46,12 @@ static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t lengt
return 0;
}
static inline void reset_shadow_queue(struct shadow_queue *queue)
{
queue->prod_idx = 0;
queue->cons_idx = 0;
}
static inline void destroy_shadow_queue(struct shadow_queue *queue)
{
kvfree(queue->buffer);
@ -57,12 +59,14 @@ static inline void destroy_shadow_queue(struct shadow_queue *queue)
static inline bool shadow_queue_full(struct shadow_queue *queue)
{
return (queue->prod_idx - queue->cons_idx) >= queue->length;
/* Do not reuse an entry until the poller has finished reading it. */
return (queue->prod_idx - smp_load_acquire(&queue->cons_idx)) >= queue->length;
}
static inline bool shadow_queue_empty(struct shadow_queue *queue)
{
return queue->prod_idx == queue->cons_idx;
/* Pair with posting's release of the initialized shadow WQE. */
return smp_load_acquire(&queue->prod_idx) == queue->cons_idx;
}
static inline void *
@ -82,34 +86,23 @@ shadow_queue_producer_entry(struct shadow_queue *queue)
static inline void *
shadow_queue_get_next_to_consume(const struct shadow_queue *queue)
{
if (queue->cons_idx == queue->next_to_complete_idx)
/* The producer publishes the WQE before advancing prod_idx. */
if (queue->cons_idx == smp_load_acquire(&queue->prod_idx))
return NULL;
return shadow_queue_get_element(queue, queue->cons_idx);
}
static inline void *
shadow_queue_get_next_to_complete(struct shadow_queue *queue)
{
if (queue->next_to_complete_idx == queue->prod_idx)
return NULL;
return shadow_queue_get_element(queue, queue->next_to_complete_idx);
}
static inline void shadow_queue_advance_producer(struct shadow_queue *queue)
{
queue->prod_idx++;
/* Publish all WQE fields to CQ polling on another CPU. */
smp_store_release(&queue->prod_idx, queue->prod_idx + 1);
}
static inline void shadow_queue_advance_consumer(struct shadow_queue *queue)
{
queue->cons_idx++;
}
static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue)
{
queue->next_to_complete_idx++;
/* Finish WC generation and queue-tail updates before allowing reuse. */
smp_store_release(&queue->cons_idx, queue->cons_idx + 1);
}
#endif

3
drivers/infiniband/hw/mana/wq.c
View File

@ -25,7 +25,8 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd,
ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr);
err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue);
err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue,
false);
if (err) {
ibdev_dbg(&mdev->ib_dev,
"Failed to create queue for create wq, %d\n", err);

170
drivers/infiniband/hw/mana/wr.c
View File

@ -7,29 +7,30 @@
#define MAX_WR_SGL_NUM (2)
static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr)
static int mana_ib_post_rq(struct mana_ib_qp *qp, const struct ib_recv_wr *wr)
{
struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem;
struct ib_sge mana_ib_dummy_sge = {.addr = 1, .length = 0, .lkey = 0};
struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp);
struct gdma_posted_wqe_info wqe_info = {0};
struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM];
struct gdma_queue *queue = ib_rq->kmem;
struct gdma_wqe_request wqe_req = {0};
struct ud_rq_shadow_wqe *shadow_wqe;
int err, i;
struct shadow_wqe_header *shadow_wqe;
int err;
if (shadow_queue_full(&qp->shadow_rq))
return -EINVAL;
if (wr->num_sge > MAX_WR_SGL_NUM)
if (wr->num_sge > MAX_RX_WQE_SGL_ENTRIES)
return -EINVAL;
for (i = 0; i < wr->num_sge; ++i) {
gdma_sgl[i].address = wr->sg_list[i].addr;
gdma_sgl[i].mem_key = wr->sg_list[i].lkey;
gdma_sgl[i].size = wr->sg_list[i].length;
}
wqe_req.num_sge = wr->num_sge;
wqe_req.sgl = gdma_sgl;
wqe_req.ib_sgl = wr->sg_list;
wqe_req.flags = GDMA_WR_IB_SGL;
if (wr->num_sge == 0) {
wqe_req.ib_sgl = &mana_ib_dummy_sge;
wqe_req.num_sge = 1;
}
err = mana_gd_post_work_request(queue, &wqe_req, &wqe_info);
if (err)
@ -37,42 +38,55 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *
shadow_wqe = shadow_queue_producer_entry(&qp->shadow_rq);
memset(shadow_wqe, 0, sizeof(*shadow_wqe));
shadow_wqe->header.opcode = IB_WC_RECV;
shadow_wqe->header.wr_id = wr->wr_id;
shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu;
shadow_wqe->wr_id = wr->wr_id;
shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset;
shadow_queue_advance_producer(&qp->shadow_rq);
mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
return 0;
}
static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr,
const struct ib_recv_wr **bad_wr)
{
struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp);
struct gdma_queue *rq = ib_rq->kmem;
bool ring_rq = false;
int err = 0;
for (; wr; wr = wr->next) {
err = mana_ib_post_rq(qp, wr);
if (unlikely(err)) {
*bad_wr = wr;
break;
}
ring_rq = true;
}
if (ring_rq)
mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), rq);
return err;
}
int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
const struct ib_recv_wr **bad_wr)
{
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
int err = 0;
for (; wr; wr = wr->next) {
switch (ibqp->qp_type) {
case IB_QPT_UD:
case IB_QPT_GSI:
err = mana_ib_post_recv_ud(qp, wr);
if (unlikely(err)) {
*bad_wr = wr;
return err;
}
break;
default:
ibdev_dbg(ibqp->device, "Posting recv wr on qp type %u is not supported\n",
ibqp->qp_type);
return -EINVAL;
}
switch (ibqp->qp_type) {
case IB_QPT_UD:
case IB_QPT_GSI:
return mana_ib_post_recv_ud(qp, wr, bad_wr);
default:
/* Unsupported QP type */
*bad_wr = wr;
return -EINVAL;
}
return err;
}
static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)
static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr *wr)
{
struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
struct mana_ib_ah *ah = container_of(wr->ah, struct mana_ib_ah, ibah);
@ -82,14 +96,12 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
struct gdma_posted_wqe_info wqe_info = {0};
struct gdma_wqe_request wqe_req = {0};
struct rdma_send_oob send_oob = {0};
struct ud_sq_shadow_wqe *shadow_wqe;
struct shadow_wqe_header *shadow_wqe;
bool signaled = qp->sq_sig_all || (wr->wr.send_flags & IB_SEND_SIGNALED);
int err, i;
if (!ndev) {
ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n",
qp->port, qp->ibqp.qp_num);
if (!ndev)
return -EINVAL;
}
if (wr->wr.opcode != IB_WR_SEND)
return -EINVAL;
@ -111,16 +123,16 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
wqe_req.num_sge = wr->wr.num_sge + 1;
wqe_req.sgl = gdma_sgl;
wqe_req.inline_oob_size = sizeof(struct rdma_send_oob);
wqe_req.inline_oob_size = INLINE_OOB_LARGE_SIZE;
wqe_req.inline_oob_data = &send_oob;
wqe_req.flags = GDMA_WR_OOB_IN_SGL;
wqe_req.client_data_unit = ib_mtu_enum_to_int(ib_mtu_int_to_enum(ndev->mtu));
send_oob.wqe_type = WQE_TYPE_UD_SEND;
send_oob.fence = !!(wr->wr.send_flags & IB_SEND_FENCE);
send_oob.signaled = !!(wr->wr.send_flags & IB_SEND_SIGNALED);
send_oob.signaled = signaled;
send_oob.solicited = !!(wr->wr.send_flags & IB_SEND_SOLICITED);
send_oob.psn = qp->ud_qp.sq_psn;
send_oob.psn = qp->sq_psn;
send_oob.ssn_or_rqpn = wr->remote_qpn;
send_oob.ud_send.remote_qkey =
qp->ibqp.qp_type == IB_QPT_GSI ? IB_QP1_QKEY : wr->remote_qkey;
@ -129,40 +141,64 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr
if (err)
return err;
qp->ud_qp.sq_psn++;
qp->sq_psn++;
shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq);
memset(shadow_wqe, 0, sizeof(*shadow_wqe));
shadow_wqe->header.opcode = IB_WC_SEND;
shadow_wqe->header.wr_id = wr->wr.wr_id;
shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu;
shadow_wqe->wr_id = wr->wr.wr_id;
shadow_wqe->flags = signaled ? 0 : MANA_WQ_NO_SIGNAL_WC;
shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu;
shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset;
shadow_queue_advance_producer(&qp->shadow_sq);
mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
return 0;
}
static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr *wr,
const struct ib_send_wr **bad_wr)
{
struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem;
unsigned long flags;
bool ring_sq = false;
bool flush_send;
int err = 0;
for (; wr; wr = wr->next) {
err = mana_ib_post_send_ud_one(qp, ud_wr(wr));
if (unlikely(err)) {
*bad_wr = wr;
break;
}
ring_sq = true;
}
if (ring_sq) {
mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue);
spin_lock_irqsave(&cq->cq_lock, flags);
flush_send = !list_empty(&qp->send_err_node);
spin_unlock_irqrestore(&cq->cq_lock, flags);
if (flush_send && cq->ibcq.comp_handler)
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
return err;
}
int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
const struct ib_send_wr **bad_wr)
{
int err = 0;
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
for (; wr; wr = wr->next) {
switch (ibqp->qp_type) {
case IB_QPT_UD:
case IB_QPT_GSI:
err = mana_ib_post_send_ud(qp, ud_wr(wr));
if (unlikely(err)) {
*bad_wr = wr;
return err;
}
break;
default:
ibdev_dbg(ibqp->device, "Posting send wr on qp type %u is not supported\n",
ibqp->qp_type);
return -EINVAL;
}
switch (ibqp->qp_type) {
case IB_QPT_UD:
case IB_QPT_GSI:
return mana_ib_post_send_ud(qp, wr, bad_wr);
default:
/* Unsupported QP type */
*bad_wr = wr;
return -EINVAL;
}
return err;
}

10
drivers/infiniband/hw/mlx4/cm.c
View File

@ -301,10 +301,20 @@ static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id,
spin_lock_irqsave(&sriov->going_down_lock, flags);
/*make sure that there is no schedule inside the scheduled work.*/
if (!sriov->is_going_down || id->scheduled_delete) {
/*
* Do not requeue timeout work that is already executing. The
* timeout handler may be waiting for id_map_lock and will free
* this entry after the caller releases it.
*/
if (id->scheduled_delete &&
!cancel_delayed_work(&id->timeout))
goto out;
id->scheduled_delete = 1;
id->rtu_timeout = rtu_timeout;
mod_delayed_work(cm_wq, &id->timeout, timeout);
}
out:
spin_unlock_irqrestore(&sriov->going_down_lock, flags);
}

17
drivers/infiniband/hw/mlx4/cq.c
View File

@ -185,9 +185,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
goto err_umem;
}
} else {
cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
cq->umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL,
ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->umem)) {
err = PTR_ERR(cq->umem);
goto err_cq;
@ -354,9 +355,10 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq
if (!cq->resize_buf)
return -ENOMEM;
cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
cq->resize_umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL,
ucmd.buf_addr,
entries * cqe_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->resize_umem)) {
err = PTR_ERR(cq->resize_umem);
goto err_buf;
@ -989,7 +991,8 @@ void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq)
* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
while (prod_index != cq->mcq.cons_index) {
--prod_index;
cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
cqe += cqe_inc;

4
drivers/infiniband/hw/mlx4/main.c
View File

@ -1169,6 +1169,10 @@ static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
struct mlx4_clock_params params;
int ret;
if (vma->vm_flags & (VM_WRITE | VM_EXEC))
return -EPERM;
vm_flags_clear(vma, VM_MAYWRITE);
ret = mlx4_get_internal_clock_params(dev->dev, &params);
if (ret)
return ret;

4
drivers/infiniband/hw/mlx4/mcg.c
View File

@ -433,7 +433,7 @@ static u16 cmp_rec(struct ib_sa_mcmember_data *src,
}
/* release group, return 1 if this was last release and group is destroyed
* timout work is canceled sync */
* timeout work is canceled sync */
static int release_group(struct mcast_group *group, int from_timeout_handler)
{
struct mlx4_ib_demux_ctx *ctx = group->demux;
@ -694,7 +694,7 @@ static void mlx4_ib_mcg_work_handler(struct work_struct *work)
cur_join_state = group->rec.scope_join_state & 0xf;
if (method == IB_MGMT_METHOD_GET_RESP) {
/* successfull join */
/* successful join */
if (!cur_join_state && resp_join_state)
--rc;
} else if (!resp_join_state)

1
drivers/infiniband/hw/mlx5/Makefile
View File

@ -6,7 +6,6 @@ mlx5_ib-y := ah.o \
cong.o \
counters.o \
cq.o \
data_direct.o \
dm.o \
dmah.o \
doorbell.o \

21
drivers/infiniband/hw/mlx5/cmd.c
View File

@ -245,24 +245,3 @@ int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid)
MLX5_SET(dealloc_uar_in, in, uid, uid);
return mlx5_cmd_exec_in(dev, dealloc_uar, in);
}
int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct,
char *out_vuid)
{
u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) +
MLX5_ST_SZ_BYTES(array1024_auto)] = {};
u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {};
char *vuid;
int err;
MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID);
MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id));
MLX5_SET(query_vuid_in, in, data_direct, data_direct);
err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
if (err)
return err;
vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid);
memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto));
return 0;
}

2
drivers/infiniband/hw/mlx5/cmd.h
View File

@ -58,6 +58,4 @@ int mlx5_cmd_mad_ifc(struct mlx5_ib_dev *dev, const void *inb, void *outb,
u16 opmod, u8 port);
int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid);
int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid);
int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct,
char *out_vuid);
#endif /* MLX5_IB_CMD_H */

14
drivers/infiniband/hw/mlx5/cq.c
View File

@ -1117,6 +1117,9 @@ int mlx5_ib_pre_destroy_cq(struct ib_cq *cq)
struct mlx5_ib_dev *dev = to_mdev(cq->device);
struct mlx5_ib_cq *mcq = to_mcq(cq);
if (!cq->uobject)
cancel_work_sync(&mcq->notify_work);
return mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
}
@ -1169,7 +1172,8 @@ void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq)
/* Now sweep backwards through the CQ, removing CQ entries
* that match our QP by copying older entries on top of them.
*/
while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) {
while (prod_index != cq->mcq.cons_index) {
--prod_index;
cqe = get_cqe(cq, prod_index & cq->ibcq.cqe);
cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64;
if (is_equal_rsn(cqe64, rsn)) {
@ -1245,9 +1249,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1)
return -EINVAL;
umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr,
(size_t)ucmd.cqe_size * entries,
IB_ACCESS_LOCAL_WRITE);
umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, ucmd.buf_addr,
(size_t)ucmd.cqe_size * entries,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
err = PTR_ERR(umem);
return err;
@ -1312,7 +1316,7 @@ static int copy_resize_cqes(struct mlx5_ib_cq *cq)
while (get_cqe_opcode(scqe64) != MLX5_CQE_RESIZE_CQ) {
dcqe = mlx5_frag_buf_get_wqe(&cq->resize_buf->fbc,
(i + 1) & cq->resize_buf->nent);
(i + 1) & (cq->resize_buf->nent - 1));
dcqe64 = dsize == 64 ? dcqe : dcqe + 64;
sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent);
memcpy(dcqe, scqe, dsize);

227
drivers/infiniband/hw/mlx5/data_direct.c
View File

@ -1,227 +0,0 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved
*/
#include "mlx5_ib.h"
#include "data_direct.h"
static LIST_HEAD(mlx5_data_direct_dev_list);
static LIST_HEAD(mlx5_data_direct_reg_list);
/*
* This mutex should be held when accessing either of the above lists
*/
static DEFINE_MUTEX(mlx5_data_direct_mutex);
struct mlx5_data_direct_registration {
struct mlx5_ib_dev *ibdev;
char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1];
struct list_head list;
};
static const struct pci_device_id mlx5_data_direct_pci_table[] = {
{ PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */
{ 0, }
};
static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev)
{
struct pci_dev *pdev = dev->pdev;
unsigned int vpd_size, kw_len;
u8 *vpd_data;
int start;
int ret;
vpd_data = pci_vpd_alloc(pdev, &vpd_size);
if (IS_ERR(vpd_data)) {
pci_err(pdev, "Unable to read VPD, err=%pe\n", vpd_data);
return PTR_ERR(vpd_data);
}
start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "VU", &kw_len);
if (start < 0) {
ret = start;
pci_err(pdev, "VU keyword not found, err=%d\n", ret);
goto end;
}
dev->vuid = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL);
ret = dev->vuid ? 0 : -ENOMEM;
end:
kfree(vpd_data);
return ret;
}
static void mlx5_data_direct_shutdown(struct pci_dev *pdev)
{
pci_disable_device(pdev);
}
static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)
{
int err;
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
if (err) {
dev_warn(&pdev->dev,
"Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err);
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
if (err) {
dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err);
return err;
}
}
dma_set_max_seg_size(&pdev->dev, SZ_2G);
return 0;
}
int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid)
{
struct mlx5_data_direct_registration *reg;
struct mlx5_data_direct_dev *dev;
reg = kzalloc_obj(*reg);
if (!reg)
return -ENOMEM;
reg->ibdev = ibdev;
strscpy(reg->vuid, vuid);
mutex_lock(&mlx5_data_direct_mutex);
list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {
if (strcmp(dev->vuid, vuid) == 0) {
mlx5_ib_data_direct_bind(ibdev, dev);
break;
}
}
/* Add the registration to its global list, to be used upon bind/unbind
* of its affiliated data direct device
*/
list_add_tail(&reg->list, &mlx5_data_direct_reg_list);
mutex_unlock(&mlx5_data_direct_mutex);
return 0;
}
void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev)
{
struct mlx5_data_direct_registration *reg;
mutex_lock(&mlx5_data_direct_mutex);
list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
if (reg->ibdev == ibdev) {
list_del(&reg->list);
kfree(reg);
goto end;
}
}
WARN_ON(true);
end:
mutex_unlock(&mlx5_data_direct_mutex);
}
static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)
{
struct mlx5_data_direct_registration *reg;
mutex_lock(&mlx5_data_direct_mutex);
list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
if (strcmp(dev->vuid, reg->vuid) == 0)
mlx5_ib_data_direct_bind(reg->ibdev, dev);
}
/* Add the data direct device to the global list, further IB devices may
* use it later as well
*/
list_add_tail(&dev->list, &mlx5_data_direct_dev_list);
mutex_unlock(&mlx5_data_direct_mutex);
}
static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev)
{
struct mlx5_data_direct_registration *reg;
mutex_lock(&mlx5_data_direct_mutex);
/* Prevent any further affiliations */
list_del(&dev->list);
list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {
if (strcmp(dev->vuid, reg->vuid) == 0)
mlx5_ib_data_direct_unbind(reg->ibdev);
}
mutex_unlock(&mlx5_data_direct_mutex);
}
static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct mlx5_data_direct_dev *dev;
int err;
dev = kzalloc_obj(*dev);
if (!dev)
return -ENOMEM;
dev->device = &pdev->dev;
dev->pdev = pdev;
pci_set_drvdata(dev->pdev, dev);
err = pci_enable_device(pdev);
if (err) {
dev_err(dev->device, "Cannot enable PCI device, err=%d\n", err);
goto err;
}
pci_set_master(pdev);
err = mlx5_data_direct_set_dma_caps(pdev);
if (err)
goto err_disable;
if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&
pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128))
dev_dbg(dev->device, "Enabling pci atomics failed\n");
err = mlx5_data_direct_vpd_get_vuid(dev);
if (err)
goto err_disable;
mlx5_data_direct_dev_reg(dev);
return 0;
err_disable:
pci_disable_device(pdev);
err:
kfree(dev);
return err;
}
static void mlx5_data_direct_remove(struct pci_dev *pdev)
{
struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev);
mlx5_data_direct_dev_unreg(dev);
pci_disable_device(pdev);
kfree(dev->vuid);
kfree(dev);
}
static struct pci_driver mlx5_data_direct_driver = {
.name = KBUILD_MODNAME,
.id_table = mlx5_data_direct_pci_table,
.probe = mlx5_data_direct_probe,
.remove = mlx5_data_direct_remove,
.shutdown = mlx5_data_direct_shutdown,
};
int mlx5_data_direct_driver_register(void)
{
return pci_register_driver(&mlx5_data_direct_driver);
}
void mlx5_data_direct_driver_unregister(void)
{
pci_unregister_driver(&mlx5_data_direct_driver);
}

23
drivers/infiniband/hw/mlx5/data_direct.h
View File

@ -1,23 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved
*/
#ifndef _MLX5_IB_DATA_DIRECT_H
#define _MLX5_IB_DATA_DIRECT_H
struct mlx5_ib_dev;
struct mlx5_data_direct_dev {
struct device *device;
struct pci_dev *pdev;
char *vuid;
struct list_head list;
};
int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid);
void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev);
int mlx5_data_direct_driver_register(void);
void mlx5_data_direct_driver_unregister(void);
#endif

48
drivers/infiniband/hw/mlx5/fs.c
View File

@ -17,6 +17,7 @@
#include <linux/mlx5/fs.h>
#include <linux/mlx5/fs_helpers.h>
#include <linux/mlx5/eswitch.h>
#include <linux/overflow.h>
#include <net/inet_ecn.h>
#include "mlx5_ib.h"
#include "counters.h"
@ -1647,36 +1648,32 @@ static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *de
struct ib_flow_attr *flow_attr,
struct mlx5_flow_destination *dst)
{
DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_wc, flows, 1);
DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_uc, flows, 1);
struct mlx5_ib_flow_handler *handler_ucast = NULL;
struct mlx5_ib_flow_handler *handler = NULL;
static struct {
struct ib_flow_spec_eth eth_flow;
struct ib_flow_attr flow_attr;
} leftovers_wc = { .flow_attr = { .num_of_specs = 1,
.size = sizeof(leftovers_wc) },
.eth_flow = {
.type = IB_FLOW_SPEC_ETH,
.size = sizeof(struct ib_flow_spec_eth),
.mask = { .dst_mac = { 0x1 } },
.val = { .dst_mac = { 0x1 } } } };
flow_attr_wc->num_of_specs = 1;
flow_attr_wc->size = sizeof(struct ib_flow_attr) +
sizeof(struct ib_flow_spec_eth);
flow_attr_wc->flows[0].eth.type = IB_FLOW_SPEC_ETH;
flow_attr_wc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth);
flow_attr_wc->flows[0].eth.mask.dst_mac[0] = 0x1;
flow_attr_wc->flows[0].eth.val.dst_mac[0] = 0x1;
static struct {
struct ib_flow_spec_eth eth_flow;
struct ib_flow_attr flow_attr;
} leftovers_uc = { .flow_attr = { .num_of_specs = 1,
.size = sizeof(leftovers_uc) },
.eth_flow = {
.type = IB_FLOW_SPEC_ETH,
.size = sizeof(struct ib_flow_spec_eth),
.mask = { .dst_mac = { 0x1 } },
.val = { .dst_mac = {} } } };
flow_attr_uc->num_of_specs = 1;
flow_attr_uc->size = sizeof(struct ib_flow_attr) +
sizeof(struct ib_flow_spec_eth);
flow_attr_uc->flows[0].eth.type = IB_FLOW_SPEC_ETH;
flow_attr_uc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth);
flow_attr_uc->flows[0].eth.mask.dst_mac[0] = 0x1;
handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst);
handler = create_flow_rule(dev, ft_prio, flow_attr_wc, dst);
if (!IS_ERR(handler) &&
flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
handler_ucast = create_flow_rule(dev, ft_prio,
&leftovers_uc.flow_attr, dst);
flow_attr_uc, dst);
if (IS_ERR(handler_ucast)) {
mlx5_del_flow_rules(handler->rule);
ft_prio->refcount--;
@ -3017,6 +3014,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
if (err)
return err;
if (priority > MLX5_IB_FLOW_LAST_PRIO)
return -EINVAL;
obj = kzalloc_obj(*obj);
if (!obj)
return -ENOMEM;
@ -3056,8 +3056,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
return 0;
destroy_res:
--ft_prio->anchor.rule_goto_table_ref;
mlx5_steering_anchor_destroy_res(ft_prio);
if (!--ft_prio->anchor.rule_goto_table_ref)
mlx5_steering_anchor_destroy_res(ft_prio);
put_flow_table:
put_flow_table(dev, ft_prio, true);
free_obj:

198
drivers/infiniband/hw/mlx5/main.c
View File

@ -25,6 +25,7 @@
#include <linux/mlx5/vport.h>
#include <linux/mlx5/fs.h>
#include <linux/mlx5/eswitch.h>
#include <linux/mlx5/data_direct.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/lag.h>
#include <linux/list.h>
@ -51,7 +52,6 @@
#include <rdma/mlx5_user_ioctl_cmds.h>
#include <rdma/ib_ucaps.h>
#include "macsec.h"
#include "data_direct.h"
#include "dmah.h"
#define UVERBS_MODULE_NAME mlx5_ib
@ -3370,7 +3370,7 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
{
struct mlx5_ib_resources *devr = &dev->devr;
struct ib_srq_init_attr attr;
struct ib_srq *s0, *s1;
struct ib_srq *s0 = NULL, *s1;
int ret = 0;
/*
@ -3388,19 +3388,27 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
if (ret)
goto unlock;
memset(&attr, 0, sizeof(attr));
attr.attr.max_sge = 1;
attr.attr.max_wr = 1;
attr.srq_type = IB_SRQT_XRC;
attr.ext.cq = devr->c0;
/*
* s0 is an XRC-type placeholder SRQ used as the default XRQN for
* XRC QPs. Skip it when XRC is absent; all devr->s0 accesses in
* qp.c are inside XRC QP paths that the verbs layer blocks before
* reaching this driver when xrc=0, so NULL is safe.
*/
if (MLX5_CAP_GEN(dev->mdev, xrc)) {
memset(&attr, 0, sizeof(attr));
attr.attr.max_sge = 1;
attr.attr.max_wr = 1;
attr.srq_type = IB_SRQT_XRC;
attr.ext.cq = devr->c0;
s0 = ib_create_srq(devr->p0, &attr);
if (IS_ERR(s0)) {
ret = PTR_ERR(s0);
mlx5_ib_err(dev,
"Couldn't create SRQ 0 for res init, err=%pe\n",
s0);
goto unlock;
s0 = ib_create_srq(devr->p0, &attr);
if (IS_ERR(s0)) {
ret = PTR_ERR(s0);
mlx5_ib_err(dev,
"Couldn't create SRQ 0 for res init, err=%pe\n",
s0);
goto unlock;
}
}
memset(&attr, 0, sizeof(attr));
@ -3414,7 +3422,8 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
mlx5_ib_err(dev,
"Couldn't create SRQ 1 for res init, err=%pe\n",
s1);
ib_destroy_srq(s0);
if (s0)
ib_destroy_srq(s0);
goto unlock;
}
@ -3431,8 +3440,11 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
struct mlx5_ib_resources *devr = &dev->devr;
int ret;
mutex_init(&devr->cq_lock);
mutex_init(&devr->srq_lock);
if (!MLX5_CAP_GEN(dev->mdev, xrc))
return -EOPNOTSUPP;
return 0;
ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
if (ret)
@ -3444,9 +3456,6 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
return ret;
}
mutex_init(&devr->cq_lock);
mutex_init(&devr->srq_lock);
return 0;
}
@ -3457,10 +3466,13 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
/* After s0/s1 init, they are not unset during the device lifetime. */
if (devr->s1) {
ib_destroy_srq(devr->s1);
ib_destroy_srq(devr->s0);
if (devr->s0)
ib_destroy_srq(devr->s0);
}
if (MLX5_CAP_GEN(dev->mdev, xrc)) {
mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
}
mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
/* After p0/c0 init, they are not unset during the device lifetime. */
if (devr->c0) {
ib_destroy_cq(devr->c0);
@ -3470,87 +3482,6 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
mutex_destroy(&devr->srq_lock);
}
static int
mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)
{
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
struct mlx5_core_dev *mdev = dev->mdev;
bool ro_supp = false;
void *mkc;
u32 mkey;
u32 pdn;
u32 *in;
int err;
err = mlx5_core_alloc_pd(mdev, &pdn);
if (err)
return err;
in = kvzalloc(inlen, GFP_KERNEL);
if (!in) {
err = -ENOMEM;
goto err;
}
MLX5_SET(create_mkey_in, in, data_direct, 1);
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
MLX5_SET(mkc, mkc, lw, 1);
MLX5_SET(mkc, mkc, lr, 1);
MLX5_SET(mkc, mkc, rw, 1);
MLX5_SET(mkc, mkc, rr, 1);
MLX5_SET(mkc, mkc, a, 1);
MLX5_SET(mkc, mkc, pd, pdn);
MLX5_SET(mkc, mkc, length64, 1);
MLX5_SET(mkc, mkc, qpn, 0xffffff);
err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
if (err)
goto err_mkey;
dev->ddr.mkey = mkey;
dev->ddr.pdn = pdn;
/* create another mkey with RO support */
if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) {
MLX5_SET(mkc, mkc, relaxed_ordering_write, 1);
ro_supp = true;
}
if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) {
MLX5_SET(mkc, mkc, relaxed_ordering_read, 1);
ro_supp = true;
}
if (ro_supp) {
err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
/* RO is defined as best effort */
if (!err) {
dev->ddr.mkey_ro = mkey;
dev->ddr.mkey_ro_valid = true;
}
}
kvfree(in);
return 0;
err_mkey:
kvfree(in);
err:
mlx5_core_dealloc_pd(mdev, pdn);
return err;
}
static void
mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev)
{
if (dev->ddr.mkey_ro_valid)
mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro);
mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
}
static u32 get_core_cap_flags(struct ib_device *ibdev,
struct mlx5_hca_vport_context *rep)
{
@ -4016,39 +3947,42 @@ static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
return false;
}
static int mlx5_ib_data_direct_event(struct notifier_block *nb,
unsigned long action, void *data)
{
struct mlx5_ib_dev *dev =
container_of(nb, struct mlx5_ib_dev, data_direct_nb);
if (action != MLX5_DATA_DIRECT_UNBIND)
return NOTIFY_DONE;
mutex_lock(&dev->data_direct_lock);
mlx5_ib_revoke_data_direct_mrs(dev);
mutex_unlock(&dev->data_direct_lock);
return NOTIFY_OK;
}
static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
{
char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
int ret;
if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
!MLX5_CAP_GEN_2(dev->mdev, query_vuid))
if (!mlx5_data_direct_supported(dev->mdev))
return 0;
ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid);
if (ret)
return ret;
ret = mlx5_ib_create_data_direct_resources(dev);
if (ret)
return ret;
INIT_LIST_HEAD(&dev->data_direct_mr_list);
ret = mlx5_data_direct_ib_reg(dev, vuid);
if (ret)
mlx5_ib_free_data_direct_resources(dev);
dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event;
ret = mlx5_data_direct_register(dev->mdev, &dev->data_direct_nb);
return ret;
}
static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
{
if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
!MLX5_CAP_GEN_2(dev->mdev, query_vuid))
if (!mlx5_data_direct_supported(dev->mdev))
return;
mlx5_data_direct_ib_unreg(dev);
mlx5_ib_free_data_direct_resources(dev);
mlx5_data_direct_unregister(dev->mdev, &dev->data_direct_nb);
}
static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
@ -5079,22 +5013,6 @@ static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
cancel_work_sync(&devr->ports[port].pkey_change_work);
}
void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
struct mlx5_data_direct_dev *dev)
{
mutex_lock(&ibdev->data_direct_lock);
ibdev->data_direct_dev = dev;
mutex_unlock(&ibdev->data_direct_lock);
}
void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev)
{
mutex_lock(&ibdev->data_direct_lock);
mlx5_ib_revoke_data_direct_mrs(ibdev);
ibdev->data_direct_dev = NULL;
mutex_unlock(&ibdev->data_direct_lock);
}
void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
const struct mlx5_ib_profile *profile,
int stage)
@ -5521,9 +5439,6 @@ static int __init mlx5_ib_init(void)
ret = mlx5r_rep_init();
if (ret)
goto rep_err;
ret = mlx5_data_direct_driver_register();
if (ret)
goto dd_err;
ret = auxiliary_driver_register(&mlx5r_mp_driver);
if (ret)
goto mp_err;
@ -5536,8 +5451,6 @@ static int __init mlx5_ib_init(void)
drv_err:
auxiliary_driver_unregister(&mlx5r_mp_driver);
mp_err:
mlx5_data_direct_driver_unregister();
dd_err:
mlx5r_rep_cleanup();
rep_err:
rcu_barrier();
@ -5550,7 +5463,6 @@ static int __init mlx5_ib_init(void)
static void __exit mlx5_ib_cleanup(void)
{
mlx5_data_direct_driver_unregister();
auxiliary_driver_unregister(&mlx5r_driver);
auxiliary_driver_unregister(&mlx5r_mp_driver);
mlx5r_rep_cleanup();

18
drivers/infiniband/hw/mlx5/mlx5_ib.h
View File

@ -24,6 +24,7 @@
#include <rdma/mlx5_user_ioctl_cmds.h>
#include <rdma/mlx5_user_ioctl_verbs.h>
#include <linux/mlx5/data_direct.h>
#include "srq.h"
#include "qp.h"
#include "macsec.h"
@ -781,13 +782,6 @@ struct mlx5_ib_port_resources {
struct work_struct pkey_change_work;
};
struct mlx5_data_direct_resources {
u32 pdn;
u32 mkey;
u32 mkey_ro;
u8 mkey_ro_valid :1;
};
struct mlx5_ib_resources {
struct ib_cq *c0;
struct mutex cq_lock;
@ -1097,8 +1091,9 @@ struct mlx5_macsec {
struct mlx5_ib_dev {
struct ib_device ib_dev;
struct mlx5_core_dev *mdev;
struct mlx5_data_direct_dev *data_direct_dev;
/* protect accessing data_direct_dev */
/* Protects data_direct_mr_list and serializes mr
* registration/deregistration with data direct device unbind.
*/
struct mutex data_direct_lock;
struct notifier_block mdev_events;
struct notifier_block sys_error_events;
@ -1131,6 +1126,7 @@ struct mlx5_ib_dev {
spinlock_t reset_flow_resource_lock;
struct list_head qp_list;
struct list_head data_direct_mr_list;
struct notifier_block data_direct_nb;
/* Array with num_ports elements */
struct mlx5_ib_port *port;
struct mlx5_sq_bfreg bfreg;
@ -1155,7 +1151,6 @@ struct mlx5_ib_dev {
u16 pkey_table_len;
u8 lag_ports;
struct mlx5_special_mkeys mkeys;
struct mlx5_data_direct_resources ddr;
#ifdef CONFIG_MLX5_MACSEC
struct mlx5_macsec macsec;
@ -1401,9 +1396,6 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
struct ib_dm_mr_attr *attr,
struct uverbs_attr_bundle *attrs);
void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
struct mlx5_data_direct_dev *dev);
void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev);
void mlx5_ib_revoke_data_direct_mrs(struct mlx5_ib_dev *dev);
#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING

7
drivers/infiniband/hw/mlx5/mr.c
View File

@ -44,7 +44,6 @@
#include "dm.h"
#include "mlx5_ib.h"
#include "umr.h"
#include "data_direct.h"
#include "dmah.h"
static int mkey_max_umr_order(struct mlx5_ib_dev *dev)
@ -587,7 +586,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
populate ? pd : dev->umrc.pd);
/* In case a data direct flow, overwrite the pdn field by its internal kernel PD */
if (umem->is_dmabuf && ksm_mode)
MLX5_SET(mkc, mkc, pd, dev->ddr.pdn);
MLX5_SET(mkc, mkc, pd, dev->mdev->data_direct->pdn);
MLX5_SET(mkc, mkc, free, !populate);
MLX5_SET(mkc, mkc, access_mode_1_0, access_mode);
@ -997,7 +996,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
return ERR_PTR(-EOPNOTSUPP);
mutex_lock(&dev->data_direct_lock);
data_direct_dev = dev->data_direct_dev;
data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct);
if (!data_direct_dev) {
ret = -EINVAL;
goto end;
@ -1006,7 +1005,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
/* If no device's 'data direct mkey' with RO flags exists
* mask it out accordingly.
*/
if (!dev->ddr.mkey_ro_valid)
if (!dev->mdev->data_direct->mkey_ro_valid)
access_flags &= ~IB_ACCESS_RELAXED_ORDERING;
crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev,
offset, length, virt_addr, fd,

6
drivers/infiniband/hw/mlx5/odp.c
View File

@ -300,7 +300,7 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni,
/*
* Iteration one - zap the HW's MTTs. The notifiers_count ensures that
* while we are doing the invalidation, no page fault will attempt to
* overwrite the same MTTs. Concurent invalidations might race us,
* overwrite the same MTTs. Concurrent invalidations might race us,
* but they will write 0s as well, so no difference in the end result.
*/
for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) {
@ -953,7 +953,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd)
int ret;
if (mr->data_direct)
pdn = dev->ddr.pdn;
pdn = dev->mdev->data_direct->pdn;
else
pdn = to_mpd(pd)->pdn;
@ -1885,8 +1885,8 @@ mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
if (!eq->core)
return 0;
mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb);
err = mlx5_eq_destroy_generic(dev->mdev, eq->core);
cancel_work_sync(&eq->work);
err = mlx5_eq_destroy_generic(dev->mdev, eq->core);
destroy_workqueue(eq->wq);
mempool_destroy(eq->pool);

32
drivers/infiniband/hw/mlx5/qp.c
View File

@ -1624,6 +1624,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
if (err)
return err;
sq->base.container_mibqp = qp;
sq->base.mqp.event = mlx5_ib_qp_event;
err = create_raw_packet_qp_sq(dev, udata, attrs, sq, in, pd,
to_mcq(init_attr->send_cq));
if (err)
@ -1635,9 +1637,6 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
resp->sqn = sq->base.mqp.qpn;
resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
}
sq->base.container_mibqp = qp;
sq->base.mqp.event = mlx5_ib_qp_event;
}
if (qp->rq.wqe_cnt) {
@ -1647,6 +1646,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
}
rq->base.container_mibqp = qp;
rq->base.mqp.event = mlx5_ib_qp_event;
if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
@ -2092,13 +2092,13 @@ static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
}
base = &qp->trans_qp.base;
base->container_mibqp = qp;
base->mqp.event = mlx5_ib_qp_event;
err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
kvfree(in);
if (err)
return err;
base->container_mibqp = qp;
base->mqp.event = mlx5_ib_qp_event;
if (MLX5_CAP_GEN(mdev, ece_support))
params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
@ -2236,14 +2236,14 @@ static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,
qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
}
base->container_mibqp = qp;
base->mqp.event = mlx5_ib_qp_event;
err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
kvfree(in);
if (err)
goto err_create;
base->container_mibqp = qp;
base->mqp.event = mlx5_ib_qp_event;
if (MLX5_CAP_GEN(mdev, ece_support))
params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
@ -2432,6 +2432,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
}
base->container_mibqp = qp;
base->mqp.event = mlx5_ib_qp_event;
if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
qp->flags & IB_QP_CREATE_SOURCE_QPN) {
qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
@ -2446,8 +2448,6 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
if (err)
goto err_create;
base->container_mibqp = qp;
base->mqp.event = mlx5_ib_qp_event;
if (MLX5_CAP_GEN(mdev, ece_support))
params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
@ -2577,14 +2577,13 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
MLX5_CAP_GEN(mdev, go_back_n))
MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);
base->container_mibqp = qp;
base->mqp.event = mlx5_ib_qp_event;
err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
kvfree(in);
if (err)
goto err_create;
base->container_mibqp = qp;
base->mqp.event = mlx5_ib_qp_event;
get_cqs(qp->type, attr->send_cq, attr->recv_cq,
&send_cq, &recv_cq);
spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
@ -3186,7 +3185,7 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
out:
if (err) {
mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
mlx5_ib_err(dev, "Create QP type %d failed, err = %d\n", qp->type, err);
return err;
}
@ -5344,11 +5343,14 @@ static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
break;
default:
mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
return;
goto out;
}
rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
}
out:
mlx5_core_res_put(&core_qp->common);
}
static int set_delay_drop(struct mlx5_ib_dev *dev)
@ -5616,6 +5618,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
rwq = kzalloc_obj(*rwq);
if (!rwq)
return ERR_PTR(-ENOMEM);
rwq->core_qp.event = mlx5_ib_wq_event;
err = prepare_user_rq(pd, init_attr, udata, rwq);
if (err)
goto err;
@ -5639,7 +5642,6 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
goto err_copy;
}
rwq->core_qp.event = mlx5_ib_wq_event;
rwq->ibwq.event_handler = init_attr->event_handler;
return &rwq->ibwq;

Some files were not shown because too many files have changed in this diff Show More