diff --git a/Documentation/ABI/stable/sysfs-class-infiniband b/Documentation/ABI/stable/sysfs-class-infiniband index 7ba116103429..726a38470e58 100644 --- a/Documentation/ABI/stable/sysfs-class-infiniband +++ b/Documentation/ABI/stable/sysfs-class-infiniband @@ -422,124 +422,6 @@ Description: information) =============== ============================================= - -sysfs interface for Intel IB driver qib ---------------------------------------- - -What: /sys/class/infiniband/qibX/version -What: /sys/class/infiniband/qibX/hw_rev -What: /sys/class/infiniband/qibX/hca_type -What: /sys/class/infiniband/qibX/board_id -What: /sys/class/infiniband/qibX/boardversion -What: /sys/class/infiniband/qibX/nctxts -What: /sys/class/infiniband/qibX/localbus_info -What: /sys/class/infiniband/qibX/tempsense -What: /sys/class/infiniband/qibX/serial -What: /sys/class/infiniband/qibX/nfreectxts -What: /sys/class/infiniband/qibX/chip_reset -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - =============== ====================================================== - version: (RO) Display version information of installed software - and drivers. - - hw_rev: (RO) Hardware revision number - - hca_type: (RO) Host channel adapter type - - board_id: (RO) Manufacturing board id - - boardversion: (RO) Current version of the chip architecture - - nctxts: (RO) Return the number of user ports (contexts) - available - - localbus_info: (RO) Human readable localbus info - - tempsense: (RO) Display temp sense registers in decimal - - serial: (RO) Serial number of the HCA - - nfreectxts: (RO) The number of free user ports (contexts) - available. - - chip_reset: (WO) Reset the chip if possible by writing - "reset" to this file. Only allowed if no user - contexts are open that use chip resources. - =============== ====================================================== - - -What: /sys/class/infiniband/qibX/ports//sl2vl/[0-15] -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - (RO) The directory contains 16 files numbered 0-15 that specify - the Service Level (SL). Listing the SL files returns the Virtual - Lane (VL) as programmed by the SL. - -What: /sys/class/infiniband/qibX/ports//CCMgtA/cc_settings_bin -What: /sys/class/infiniband/qibX/ports//CCMgtA/cc_table_bin -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - Per-port congestion control. Both are binary attributes. - - =============== ================================================ - cc_table_bin (RO) Congestion control table size followed by - table entries. - - cc_settings_bin (RO) Congestion settings: port control, control - map and an array of 16 entries for the - congestion entries - increase, timer, event log - trigger threshold and the minimum injection rate - delay. - =============== ================================================ - -What: /sys/class/infiniband/qibX/ports//linkstate/loopback -What: /sys/class/infiniband/qibX/ports//linkstate/led_override -What: /sys/class/infiniband/qibX/ports//linkstate/hrtbt_enable -What: /sys/class/infiniband/qibX/ports//linkstate/status -What: /sys/class/infiniband/qibX/ports//linkstate/status_str -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - [to be documented] - - =============== =============================================== - loopback: (WO) - led_override: (WO) - hrtbt_enable: (RW) - status: (RO) - - status_str: (RO) Displays information about the link state, - possible cable/switch problems, and hardware - errors. Possible states are- "Initted", - "Present", "IB_link_up", "IB_configured" or - "Fatal_Hardware_Error". - =============== =============================================== - -What: /sys/class/infiniband/qibX/ports//diag_counters/rc_resends -What: /sys/class/infiniband/qibX/ports//diag_counters/seq_naks -What: /sys/class/infiniband/qibX/ports//diag_counters/rdma_seq -What: /sys/class/infiniband/qibX/ports//diag_counters/rnr_naks -What: /sys/class/infiniband/qibX/ports//diag_counters/other_naks -What: /sys/class/infiniband/qibX/ports//diag_counters/rc_timeouts -What: /sys/class/infiniband/qibX/ports//diag_counters/look_pkts -What: /sys/class/infiniband/qibX/ports//diag_counters/pkt_drops -What: /sys/class/infiniband/qibX/ports//diag_counters/dma_wait -What: /sys/class/infiniband/qibX/ports//diag_counters/unaligned -Date: May, 2010 -KernelVersion: v2.6.35 -Contact: linux-rdma@vger.kernel.org -Description: - [to be documented] - - sysfs interface for Mellanox Connect-IB HCA driver mlx5 ------------------------------------------------------- diff --git a/Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml b/Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml new file mode 100644 index 000000000000..d987840f417c --- /dev/null +++ b/Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml @@ -0,0 +1,173 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/infiniband/hisilicon,hns-roce-v1.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Hisilicon HNS RoCE + +maintainers: + - Chengchang Tang + - Junxian Huang + +description: + Hisilicon RoCE engine is a part of network subsystem. It works depending on + other part of network subsystem, such as gmac and dsa fabric. + +properties: + compatible: + const: hisilicon,hns-roce-v1 + + reg: + maxItems: 1 + + '#address-cells': + const: 2 + + '#size-cells': + const: 2 + + dma-coherent: true + + interrupts: + items: + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Completion event IRQ + - description: Async event IRQ + - description: Event overflow IRQ + + interrupt-names: + items: + - const: hns-roce-comp-0 + - const: hns-roce-comp-1 + - const: hns-roce-comp-2 + - const: hns-roce-comp-3 + - const: hns-roce-comp-4 + - const: hns-roce-comp-5 + - const: hns-roce-comp-6 + - const: hns-roce-comp-7 + - const: hns-roce-comp-8 + - const: hns-roce-comp-9 + - const: hns-roce-comp-10 + - const: hns-roce-comp-11 + - const: hns-roce-comp-12 + - const: hns-roce-comp-13 + - const: hns-roce-comp-14 + - const: hns-roce-comp-15 + - const: hns-roce-comp-16 + - const: hns-roce-comp-17 + - const: hns-roce-comp-18 + - const: hns-roce-comp-19 + - const: hns-roce-comp-20 + - const: hns-roce-comp-21 + - const: hns-roce-comp-22 + - const: hns-roce-comp-23 + - const: hns-roce-comp-24 + - const: hns-roce-comp-25 + - const: hns-roce-comp-26 + - const: hns-roce-comp-27 + - const: hns-roce-comp-28 + - const: hns-roce-comp-29 + - const: hns-roce-comp-30 + - const: hns-roce-comp-31 + - const: hns-roce-async + - const: hns-roce-common + + eth-handle: + description: References to one or more Ethernet devices. + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + maxItems: 1 + + dsaf-handle: + description: Reference to a dsaf device. + $ref: /schemas/types.yaml#/definitions/phandle + + node-guid: + description: A number that uniquely identifies a device or component. + $ref: /schemas/types.yaml#/definitions/uint8-array + +required: + - compatible + - reg + - '#address-cells' + - '#size-cells' + - eth-handle + - dsaf-handle + - node-guid + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + infiniband@c4000000 { + compatible = "hisilicon,hns-roce-v1"; + reg = <0x0 0xc4000000 0x0 0x100000>; + #address-cells = <2>; + #size-cells = <2>; + dma-coherent; + eth-handle = <ð2 ð3 ð4 ð5 ð6 ð7>; + dsaf-handle = <&soc0_dsa>; + node-guid = [00 9A CD 00 00 01 02 03]; + interrupts = <722 1>, <723 1>, <724 1>, <725 1>, + <726 1>, <727 1>, <728 1>, <729 1>, + <730 1>, <731 1>, <732 1>, <733 1>, + <734 1>, <735 1>, <736 1>, <737 1>, + <738 1>, <739 1>, <740 1>, <741 1>, + <742 1>, <743 1>, <744 1>, <745 1>, + <746 1>, <747 1>, <748 1>, <749 1>, + <750 1>, <751 1>, <752 1>, <753 1>, + <785 1>, <754 4>; + interrupt-names = "hns-roce-comp-0", "hns-roce-comp-1", + "hns-roce-comp-2", "hns-roce-comp-3", + "hns-roce-comp-4", "hns-roce-comp-5", + "hns-roce-comp-6", "hns-roce-comp-7", + "hns-roce-comp-8", "hns-roce-comp-9", + "hns-roce-comp-10", "hns-roce-comp-11", + "hns-roce-comp-12", "hns-roce-comp-13", + "hns-roce-comp-14", "hns-roce-comp-15", + "hns-roce-comp-16", "hns-roce-comp-17", + "hns-roce-comp-18", "hns-roce-comp-19", + "hns-roce-comp-20", "hns-roce-comp-21", + "hns-roce-comp-22", "hns-roce-comp-23", + "hns-roce-comp-24", "hns-roce-comp-25", + "hns-roce-comp-26", "hns-roce-comp-27", + "hns-roce-comp-28", "hns-roce-comp-29", + "hns-roce-comp-30", "hns-roce-comp-31", + "hns-roce-async", "hns-roce-common"; + }; + }; diff --git a/Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt b/Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt deleted file mode 100644 index be31cf05cd2e..000000000000 --- a/Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt +++ /dev/null @@ -1,108 +0,0 @@ -Hisilicon RoCE DT description - -Hisilicon RoCE engine is a part of network subsystem. -It works depending on other part of network subsystem, such as gmac and -dsa fabric. - -Additional properties are described here: - -Required properties: -- compatible: Should contain "hisilicon,hns-roce-v1". -- reg: Physical base address of the RoCE driver and -length of memory mapped region. -- eth-handle: phandle, specifies a reference to a node -representing a ethernet device. -- dsaf-handle: phandle, specifies a reference to a node -representing a dsaf device. -- node_guid: a number that uniquely identifies a device or component -- #address-cells: must be 2 -- #size-cells: must be 2 -Optional properties: -- dma-coherent: Present if DMA operations are coherent. -- interrupts: should contain 32 completion event irq,1 async event irq -and 1 event overflow irq. -- interrupt-names:should be one of 34 irqs for roce device - - hns-roce-comp-0 ~ hns-roce-comp-31: 32 complete event irq - - hns-roce-async: 1 async event irq - - hns-roce-common: named common exception warning irq -Example: - infiniband@c4000000 { - compatible = "hisilicon,hns-roce-v1"; - reg = <0x0 0xc4000000 0x0 0x100000>; - dma-coherent; - eth-handle = <ð2 ð3 ð4 ð5 ð6 ð7>; - dsaf-handle = <&soc0_dsa>; - node-guid = [00 9A CD 00 00 01 02 03]; - #address-cells = <2>; - #size-cells = <2>; - interrupt-parent = <&mbigen_dsa>; - interrupts = <722 1>, - <723 1>, - <724 1>, - <725 1>, - <726 1>, - <727 1>, - <728 1>, - <729 1>, - <730 1>, - <731 1>, - <732 1>, - <733 1>, - <734 1>, - <735 1>, - <736 1>, - <737 1>, - <738 1>, - <739 1>, - <740 1>, - <741 1>, - <742 1>, - <743 1>, - <744 1>, - <745 1>, - <746 1>, - <747 1>, - <748 1>, - <749 1>, - <750 1>, - <751 1>, - <752 1>, - <753 1>, - <785 1>, - <754 4>; - - interrupt-names = "hns-roce-comp-0", - "hns-roce-comp-1", - "hns-roce-comp-2", - "hns-roce-comp-3", - "hns-roce-comp-4", - "hns-roce-comp-5", - "hns-roce-comp-6", - "hns-roce-comp-7", - "hns-roce-comp-8", - "hns-roce-comp-9", - "hns-roce-comp-10", - "hns-roce-comp-11", - "hns-roce-comp-12", - "hns-roce-comp-13", - "hns-roce-comp-14", - "hns-roce-comp-15", - "hns-roce-comp-16", - "hns-roce-comp-17", - "hns-roce-comp-18", - "hns-roce-comp-19", - "hns-roce-comp-20", - "hns-roce-comp-21", - "hns-roce-comp-22", - "hns-roce-comp-23", - "hns-roce-comp-24", - "hns-roce-comp-25", - "hns-roce-comp-26", - "hns-roce-comp-27", - "hns-roce-comp-28", - "hns-roce-comp-29", - "hns-roce-comp-30", - "hns-roce-comp-31", - "hns-roce-async", - "hns-roce-common"; - }; diff --git a/MAINTAINERS b/MAINTAINERS index f802c67cc581..5b70859d117b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11935,7 +11935,7 @@ M: Chengchang Tang M: Junxian Huang L: linux-rdma@vger.kernel.org S: Maintained -F: Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt +F: Documentation/devicetree/bindings/infiniband/hisilicon,hns-roce-v1.yaml F: drivers/infiniband/hw/hns/ HISILICON SAS Controller @@ -18132,7 +18132,7 @@ F: include/linux/cciss*.h F: include/uapi/linux/cciss*.h MICROSOFT MANA RDMA DRIVER -M: Long Li +M: Wei Hu M: Konstantin Taranov L: linux-rdma@vger.kernel.org L: linux-hyperv@vger.kernel.org diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index 73170b15fc3d..337a49d1acf7 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -4823,6 +4823,29 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data, } EXPORT_SYMBOL(rdma_reject); +static int cma_force_disconnect(struct rdma_id_private *id_priv) +{ + struct cma_work *work; + + work = kzalloc_obj(*work); + if (!work) + return -ENOMEM; + + INIT_WORK(&work->work, cma_work_handler); + work->old_state = RDMA_CM_CONNECT; + work->new_state = RDMA_CM_DISCONNECT; + work->event.event = RDMA_CM_EVENT_DISCONNECTED; + work->event.status = -ECONNABORTED; + + cma_id_get(id_priv); + work->id = id_priv; + + trace_cm_force_disconnect(id_priv); + + queue_work(cma_wq, &work->work); + return 0; +} + int rdma_disconnect(struct rdma_cm_id *id) { struct rdma_id_private *id_priv; @@ -4838,12 +4861,15 @@ int rdma_disconnect(struct rdma_cm_id *id) goto out; /* Initiate or respond to a disconnect. */ trace_cm_disconnect(id_priv); - if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) { - if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) - trace_cm_sent_drep(id_priv); - } else { + if (!ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) { trace_cm_sent_dreq(id_priv); + goto out; } + if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) { + trace_cm_sent_drep(id_priv); + goto out; + } + ret = cma_force_disconnect(id_priv); } else if (rdma_cap_iw_cm(id->device, id->port_num)) { ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); } else diff --git a/drivers/infiniband/core/cma_trace.h b/drivers/infiniband/core/cma_trace.h index 3456d5f3aa47..3d05d8e80856 100644 --- a/drivers/infiniband/core/cma_trace.h +++ b/drivers/infiniband/core/cma_trace.h @@ -61,6 +61,7 @@ DEFINE_CMA_FSM_EVENT(send_sidr_rep); DEFINE_CMA_FSM_EVENT(disconnect); DEFINE_CMA_FSM_EVENT(sent_drep); DEFINE_CMA_FSM_EVENT(sent_dreq); +DEFINE_CMA_FSM_EVENT(force_disconnect); DEFINE_CMA_FSM_EVENT(id_destroy); TRACE_EVENT(cm_id_attach, diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index 7a3ed5ecac00..29af9fe860e2 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -893,7 +893,7 @@ static int setup_port_data(struct ib_device *device) * ib_port_immutable_read() - Read rdma port's immutable data * @dev: IB device * @port: port number whose immutable data to read. It starts with index 1 and - * valid upto including rdma_end_port(). + * valid up to including rdma_end_port(). */ const struct ib_port_immutable* ib_port_immutable_read(struct ib_device *dev, unsigned int port) @@ -3067,7 +3067,7 @@ int ib_add_sub_device(struct ib_device *parent, sub->parent = parent; mutex_lock(&parent->subdev_lock); - list_add_tail(&parent->subdev_list_head, &sub->subdev_list); + list_add_tail(&sub->subdev_list, &parent->subdev_list_head); mutex_unlock(&parent->subdev_lock); return ret; diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c index 41c84ffe8c09..74d9c006bad5 100644 --- a/drivers/infiniband/core/ib_core_uverbs.c +++ b/drivers/infiniband/core/ib_core_uverbs.c @@ -244,16 +244,8 @@ void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry) entry->driver_removed = true; xa_unlock(&entry->ucontext->mmap_xa); list_for_each_entry_safe(uverbs_dmabuf, tmp, &entry->dmabufs, dmabufs_elm) { - dma_resv_lock(uverbs_dmabuf->dmabuf->resv, NULL); list_del(&uverbs_dmabuf->dmabufs_elm); - uverbs_dmabuf->revoked = true; - dma_buf_invalidate_mappings(uverbs_dmabuf->dmabuf); - dma_resv_wait_timeout(uverbs_dmabuf->dmabuf->resv, - DMA_RESV_USAGE_BOOKKEEP, false, - MAX_SCHEDULE_TIMEOUT); - dma_resv_unlock(uverbs_dmabuf->dmabuf->resv); - kref_put(&uverbs_dmabuf->kref, ib_uverbs_dmabuf_done); - wait_for_completion(&uverbs_dmabuf->comp); + ib_uverbs_dmabuf_revoke(uverbs_dmabuf); } mutex_unlock(&entry->dmabufs_lock); @@ -535,18 +527,6 @@ int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs) } EXPORT_SYMBOL(uverbs_destroy_def_handler); -/* - * When calling a destroy function during an error unwind we need to pass in - * the udata that is sanitized of all user arguments. Ie from the driver - * perspective it looks like no udata was passed. - */ -struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) -{ - attrs->driver_udata = (struct ib_udata){}; - return &attrs->driver_udata; -} -EXPORT_SYMBOL_NS_GPL(uverbs_get_cleared_udata, "rdma_core"); - /** * _uverbs_alloc() - Quickly allocate memory for use with a bundle * @bundle: The bundle diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c index 0b7246ec559e..4b05867dad89 100644 --- a/drivers/infiniband/core/iwcm.c +++ b/drivers/infiniband/core/iwcm.c @@ -936,7 +936,7 @@ static void cm_disconnect_handler(struct iwcm_id_private *cm_id_priv, /* * CM_ID <-- IDLE * - * If in the ESTBLISHED or CLOSING states, the QP will have have been + * If in the ESTBLISHED or CLOSING states, the QP will have been * moved by the provider to the ERR state. Disassociate the CM_ID from * the QP, move to IDLE, and remove the 'connected' reference. * diff --git a/drivers/infiniband/core/lag.c b/drivers/infiniband/core/lag.c index 00fe241737ff..651c868aa04e 100644 --- a/drivers/infiniband/core/lag.c +++ b/drivers/infiniband/core/lag.c @@ -19,6 +19,11 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev, u8 smac[ETH_ALEN]; bool is_ipv4; int hdr_len; + int ret; + + ret = rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, NULL, smac); + if (ret) + return ERR_PTR(ret); is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw); hdr_len = ETH_HLEN + sizeof(struct udphdr) + LL_RESERVED_SPACE(netdev); @@ -26,7 +31,7 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev, skb = alloc_skb(hdr_len, flags); if (!skb) - return NULL; + return ERR_PTR(-ENOMEM); skb->dev = netdev; skb_reserve(skb, hdr_len); @@ -59,7 +64,7 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev, ip6h->version = 6; ip6h->nexthdr = IPPROTO_UDP; memcpy(&ip6h->flow_lbl, &ah_attr->grh.flow_label, - sizeof(*ip6h->flow_lbl)); + sizeof(ip6h->flow_lbl)); memcpy(&ip6h->saddr, ah_attr->grh.sgid_attr->gid.raw, sizeof(struct in6_addr)); memcpy(&ip6h->daddr, ah_attr->grh.dgid.raw, @@ -70,7 +75,6 @@ static struct sk_buff *rdma_build_skb(struct net_device *netdev, skb_reset_mac_header(skb); eth = eth_hdr(skb); skb->protocol = eth->h_proto = htons(is_ipv4 ? ETH_P_IP : ETH_P_IPV6); - rdma_read_gid_l2_fields(ah_attr->grh.sgid_attr, NULL, smac); memcpy(eth->h_source, smac, ETH_ALEN); memcpy(eth->h_dest, ah_attr->roce.dmac, ETH_ALEN); @@ -86,8 +90,8 @@ static struct net_device *rdma_get_xmit_slave_udp(struct ib_device *device, struct sk_buff *skb; skb = rdma_build_skb(master, ah_attr, flags); - if (!skb) - return ERR_PTR(-ENOMEM); + if (IS_ERR(skb)) + return ERR_CAST(skb); rcu_read_lock(); slave = netdev_get_xmit_slave(master, skb, diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index 3c91f00d09c6..8124e0b5a87d 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -2342,6 +2342,8 @@ static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc) recv->header.mapping, mad_priv_dma_size(recv), DMA_FROM_DEVICE); + if (unlikely(wc->byte_len < sizeof(struct ib_grh))) + goto out; /* Setup MAD receive work completion from "normal" work completion */ recv->header.wc = *wc; diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c index 17c4c52a19e4..f05e9ce93c33 100644 --- a/drivers/infiniband/core/mad_rmpp.c +++ b/drivers/infiniband/core/mad_rmpp.c @@ -245,12 +245,15 @@ static void recv_timeout_handler(struct work_struct *work) struct ib_mad_recv_wc *rmpp_wc; unsigned long flags; - spin_lock_irqsave(&rmpp_recv->agent->lock, flags); + spin_lock_irqsave(&rmpp_recv->lock, flags); if (rmpp_recv->state != RMPP_STATE_ACTIVE) { - spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags); + spin_unlock_irqrestore(&rmpp_recv->lock, flags); return; } rmpp_recv->state = RMPP_STATE_TIMEOUT; + spin_unlock_irqrestore(&rmpp_recv->lock, flags); + + spin_lock_irqsave(&rmpp_recv->agent->lock, flags); list_del(&rmpp_recv->list); spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags); diff --git a/drivers/infiniband/core/multicast.c b/drivers/infiniband/core/multicast.c index bea7df3dd8f3..f549fd928c1c 100644 --- a/drivers/infiniband/core/multicast.c +++ b/drivers/infiniband/core/multicast.c @@ -189,7 +189,10 @@ static void release_group(struct mcast_group *group) spin_lock_irqsave(&port->lock, flags); if (atomic_dec_and_test(&group->refcount)) { - rb_erase(&group->node, &port->table); + + if (!RB_EMPTY_NODE(&group->node)) + rb_erase(&group->node, &port->table); + spin_unlock_irqrestore(&port->lock, flags); kfree(group); deref_port(port); @@ -533,6 +536,8 @@ static void join_handler(int status, struct ib_sa_mcmember_rec *rec, group->rec = *rec; if (mgids_changed) { rb_erase(&group->node, &group->port->table); + + RB_CLEAR_NODE(&group->node); is_mgid0 = !memcmp(&mgid0, &group->rec.mgid, sizeof(mgid0)); mcast_insert(group->port, group, is_mgid0); diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 4e8fbee34745..849a5db54e1c 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -700,7 +700,8 @@ static int fill_res_cq_entry(struct sk_buff *msg, bool has_cap_net_admin, if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CQN, res->id)) return -EMSGSIZE; - if (!rdma_is_kernel_res(res) && + + if (cq->uobject && cq->uobject->uevent.uobject.context && nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, cq->uobject->uevent.uobject.context->res.id)) return -EMSGSIZE; @@ -790,7 +791,7 @@ static int fill_res_pd_entry(struct sk_buff *msg, bool has_cap_net_admin, if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, res->id)) goto err; - if (!rdma_is_kernel_res(res) && + if (pd->uobject && pd->uobject->context && nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_CTXN, pd->uobject->context->res.id)) goto err; @@ -1895,8 +1896,10 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh, mutex_lock(&nldev_dellink_mutex); err = device->link_ops->dellink(device); mutex_unlock(&nldev_dellink_mutex); - if (err) + if (err) { + ib_device_put(device); return err; + } } ib_unregister_device_and_put(device); diff --git a/drivers/infiniband/core/rdma_core.c b/drivers/infiniband/core/rdma_core.c index a7cbe643e33c..b32e5445601b 100644 --- a/drivers/infiniband/core/rdma_core.c +++ b/drivers/infiniband/core/rdma_core.c @@ -734,6 +734,7 @@ void rdma_assign_uobject(struct ib_uobject *to_uobj, struct ib_uobject *new_uobj * If this fails then the uobject is still completely valid (though with * a new ID) and we leak it until context close. */ + uverbs_get_cleared_udata(attrs); uverbs_destroy_uobject(to_uobj, RDMA_REMOVE_DESTROY, attrs); } EXPORT_SYMBOL_NS_GPL(rdma_assign_uobject, "rdma_core"); @@ -750,6 +751,7 @@ void rdma_alloc_abort_uobject(struct ib_uobject *uobj, int ret; if (hw_obj_valid) { + uverbs_get_cleared_udata(attrs); ret = uobj->uapi_object->type_class->destroy_hw( uobj, RDMA_REMOVE_ABORT, attrs); /* diff --git a/drivers/infiniband/core/rdma_core.h b/drivers/infiniband/core/rdma_core.h index 2b91e8527287..9de14e625dd3 100644 --- a/drivers/infiniband/core/rdma_core.h +++ b/drivers/infiniband/core/rdma_core.h @@ -71,14 +71,19 @@ int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx); void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile); -#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) -struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs); -#else -static inline struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) +/* + * When calling a destroy function during an error unwind we need to pass in + * the udata that is sanitized of all user arguments. Ie from the driver + * perspective it looks like no udata was passed. + */ +static inline struct ib_udata * +uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs) { - return NULL; + if (!attrs) + return NULL; + attrs->driver_udata = (struct ib_udata){}; + return &attrs->driver_udata; } -#endif /* * This is the runtime description of the uverbs API, used by the syscall diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index f89a81dad72f..a8b838fdc7b5 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -250,8 +250,7 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) ret = xa_insert(&rt->xa, res->id, res, GFP_KERNEL); if (ret) res->id = 0; - - if (qp->qp_type >= IB_QPT_DRIVER) + else if (qp->qp_type >= IB_QPT_DRIVER) xa_set_mark(&rt->xa, res->id, RESTRACK_DD); } else if (res->type == RDMA_RESTRACK_COUNTER) { /* Special case to ensure that cntn points to right counter */ diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c index 69df8b703cbd..4cd861b5d953 100644 --- a/drivers/infiniband/core/sysfs.c +++ b/drivers/infiniband/core/sysfs.c @@ -996,8 +996,8 @@ static int setup_hw_port_stats(struct ib_port *port, data->stats->num_counters); if (ret != data->stats->num_counters) { if (WARN_ON(ret >= 0)) - return -EINVAL; - return ret; + ret = -EINVAL; + goto err_free_data; } data->stats->timestamp = jiffies; @@ -1031,6 +1031,11 @@ static int setup_hw_port_stats(struct ib_port *port, port->hw_stats_data = data; return 0; + +err_free_data: + rdma_free_hw_stats_struct(data->stats); + kfree(data); + return ret; } struct rdma_hw_stats *ib_get_hw_stats_port(struct ib_device *ibdev, diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index 88110b9661f5..b1ab4f0359b0 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -55,7 +55,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d if (dirty) ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt, - DMA_BIDIRECTIONAL, umem->dma_attrs); + umem->dma_dir, umem->dma_attrs); for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) { unpin_user_page_range_dirty_lock(sg_page(sg), @@ -159,9 +159,15 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, } EXPORT_SYMBOL(ib_umem_find_best_pgsz); +static inline enum dma_data_direction ib_access_dma_dir(int access) +{ + return ib_access_writable(access) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; +} + static struct ib_umem *__ib_umem_get_va(struct ib_device *device, unsigned long addr, size_t size, - int access) + int access, + enum dma_data_direction dir) { struct ib_umem *umem; struct page **page_list; @@ -202,6 +208,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, */ umem->iova = addr; umem->writable = ib_access_writable(access); + umem->dma_dir = dir; umem->owning_mm = mm = current->mm; umem->dma_attrs = DMA_ATTR_REQUIRE_COHERENT; if (access & IB_ACCESS_RELAXED_ORDERING) @@ -261,7 +268,7 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, } ret = ib_dma_map_sgtable_attrs(device, &umem->sgt_append.sgt, - DMA_BIDIRECTIONAL, umem->dma_attrs); + dir, umem->dma_attrs); if (ret) goto umem_release; goto out; @@ -279,17 +286,16 @@ static struct ib_umem *__ib_umem_get_va(struct ib_device *device, return ret ? ERR_PTR(ret) : umem; } -/** - * ib_umem_get_desc - Pin a umem from a buffer descriptor. - * @device: IB device. - * @desc: buffer descriptor (VA or DMABUF). - * @access: IB access flags. +/* + * __ib_umem_get_desc_dir - core implementation for ib_umem_get_desc(). * - * Return: caller-owned umem on success, ERR_PTR(...) on error. + * @dir applies to VA buffers only; dmabuf direction is managed by the + * dmabuf subsystem and this argument is ignored for that type. */ -struct ib_umem *ib_umem_get_desc(struct ib_device *device, - const struct ib_uverbs_buffer_desc *desc, - int access) +static struct ib_umem * +__ib_umem_get_desc_dir(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + int access, enum dma_data_direction dir) { struct ib_umem_dmabuf *umem_dmabuf; @@ -310,11 +316,27 @@ struct ib_umem *ib_umem_get_desc(struct ib_device *device, return &umem_dmabuf->umem; case IB_UVERBS_BUFFER_TYPE_VA: return __ib_umem_get_va(device, desc->addr, desc->length, - access); + access, dir); default: return ERR_PTR(-EINVAL); } } + +/** + * ib_umem_get_desc - Pin a umem from a buffer descriptor. + * @device: IB device. + * @desc: buffer descriptor (VA or DMABUF). + * @access: IB access flags. + * + * Return: caller-owned umem on success, ERR_PTR(...) on error. + */ +struct ib_umem *ib_umem_get_desc(struct ib_device *device, + const struct ib_uverbs_buffer_desc *desc, + int access) +{ + return __ib_umem_get_desc_dir(device, desc, access, + ib_access_dma_dir(access)); +} EXPORT_SYMBOL(ib_umem_get_desc); /* @@ -376,11 +398,12 @@ static int ib_umem_resolve_desc(const struct uverbs_attr_bundle *attrs, static struct ib_umem * ib_umem_get_desc_check(struct ib_device *device, const struct ib_uverbs_buffer_desc *desc, - size_t min_size, int access) + size_t min_size, int access, + enum dma_data_direction dir) { struct ib_umem *umem; - umem = ib_umem_get_desc(device, desc, access); + umem = __ib_umem_get_desc_dir(device, desc, access, dir); if (IS_ERR(umem)) return umem; if (umem->length < min_size) { @@ -401,7 +424,7 @@ static struct ib_umem * ib_umem_get_from_attrs(struct ib_device *device, const struct uverbs_attr_bundle *attrs, u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler, - size_t size, int access) + size_t size, int access, enum dma_data_direction dir) { struct ib_uverbs_buffer_desc desc = {}; int ret; @@ -411,7 +434,7 @@ ib_umem_get_from_attrs(struct ib_device *device, return NULL; if (ret) return ERR_PTR(ret); - return ib_umem_get_desc_check(device, &desc, size, access); + return ib_umem_get_desc_check(device, &desc, size, access, dir); } /* @@ -431,7 +454,8 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device, const struct uverbs_attr_bundle *attrs, u16 attr_id, ib_umem_buf_desc_filler_t legacy_filler, - u64 addr, size_t size, int access) + u64 addr, size_t size, int access, + enum dma_data_direction dir) { struct ib_uverbs_buffer_desc desc = {}; int ret; @@ -445,7 +469,7 @@ ib_umem_get_from_attrs_or_va(struct ib_device *device, }; else if (ret) return ERR_PTR(ret); - return ib_umem_get_desc_check(device, &desc, size, access); + return ib_umem_get_desc_check(device, &desc, size, access, dir); } /** @@ -464,7 +488,7 @@ struct ib_umem *ib_umem_get_attr(struct ib_device *device, u16 attr_id, size_t size, int access) { return ib_umem_get_from_attrs(device, attrs, attr_id, NULL, size, - access); + access, ib_access_dma_dir(access)); } EXPORT_SYMBOL(ib_umem_get_attr); @@ -503,7 +527,8 @@ struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, int access) { return ib_umem_get_from_attrs_or_va(device, attrs, attr_id, NULL, addr, - size, access); + size, access, + ib_access_dma_dir(access)); } EXPORT_SYMBOL(ib_umem_get_attr_or_va); @@ -569,6 +594,9 @@ static int uverbs_create_cq_get_buffer_desc(const struct uverbs_attr_bundle *att * must arrange its own backing (typically an in-kernel allocation) * when no source is available. * + * The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it + * and the CPU only reads. + * * Return: caller-owned umem on success; NULL when no source supplied * a buffer; ERR_PTR(...) on error. */ @@ -579,7 +607,7 @@ struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, return ib_umem_get_from_attrs(device, attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, uverbs_create_cq_get_buffer_desc, - size, access); + size, access, DMA_FROM_DEVICE); } EXPORT_SYMBOL(ib_umem_get_cq_buf); @@ -595,6 +623,9 @@ EXPORT_SYMBOL(ib_umem_get_cq_buf); * Like ib_umem_get_cq_buf(), but pins @addr/@size when neither the * UMEM attribute nor the legacy CQ buffer attributes are supplied. * + * The buffer is mapped DMA_FROM_DEVICE: the NIC writes CQEs into it + * and the CPU only reads. + * * See ib_umem_get_attr_or_va() for the note on @size's dual role and * the migration path for drivers that would distinguish a user-supplied * length from a driver-computed minimum. @@ -608,7 +639,7 @@ struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, return ib_umem_get_from_attrs_or_va(device, attrs, UVERBS_ATTR_CREATE_CQ_BUF_UMEM, uverbs_create_cq_get_buffer_desc, - addr, size, access); + addr, size, access, DMA_FROM_DEVICE); } EXPORT_SYMBOL(ib_umem_get_cq_buf_or_va); diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h index c64dd6b94e10..728e09b41992 100644 --- a/drivers/infiniband/core/uverbs.h +++ b/drivers/infiniband/core/uverbs.h @@ -42,6 +42,8 @@ #include #include #include +#include +#include #include #include @@ -135,6 +137,13 @@ struct ib_uverbs_completion_event_file { struct ib_uverbs_dmabuf_file { struct ib_uobject uobj; + /* + * Serializes users of dmabuf with the dma_buf's release, which sets it + * to NULL. The uobject can outlive the dma_buf while it waits to be + * destroyed. The map and unmap callbacks use attachment->dmabuf + * instead of taking this lock. + */ + struct mutex dmabuf_release_lock; struct dma_buf *dmabuf; struct list_head dmabufs_elm; struct rdma_user_mmap_entry *mmap_entry; @@ -359,6 +368,34 @@ static inline void ib_uverbs_dmabuf_done(struct kref *kref) complete(&priv->comp); } +/* + * Revoke the dmabuf and wait for all of its DMA mappings to be gone. Must be + * called with the mmap entry's dmabufs_lock held, after the dmabuf has been + * removed from the entry's dmabufs list, so it runs once per dmabuf. Lock + * order is dmabufs_lock, dmabuf_release_lock, then the dma_resv lock. + */ +static inline void ib_uverbs_dmabuf_revoke(struct ib_uverbs_dmabuf_file *priv) +{ + mutex_lock(&priv->dmabuf_release_lock); + if (priv->dmabuf) { + dma_resv_lock(priv->dmabuf->resv, NULL); + priv->revoked = true; + dma_buf_invalidate_mappings(priv->dmabuf); + dma_resv_wait_timeout(priv->dmabuf->resv, + DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_unlock(priv->dmabuf->resv); + } else { + /* A released dma_buf has no attachments left to invalidate */ + priv->revoked = true; + } + mutex_unlock(&priv->dmabuf_release_lock); + + kref_put(&priv->kref, ib_uverbs_dmabuf_done); + /* Wait till all DMA unmaps are completed */ + wait_for_completion(&priv->comp); +} + int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile, enum rdma_remove_reason reason); diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index ce0a7de00405..db65032e80b4 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -272,6 +272,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( return ret; ret = ib_init_ucontext(attrs); if (ret) { + rdma_restrack_put(&attrs->context->res); kfree(attrs->context); attrs->context = NULL; return ret; diff --git a/drivers/infiniband/core/uverbs_std_types_dmabuf.c b/drivers/infiniband/core/uverbs_std_types_dmabuf.c index 2411ebee69e2..8d8c997a4883 100644 --- a/drivers/infiniband/core/uverbs_std_types_dmabuf.c +++ b/drivers/infiniband/core/uverbs_std_types_dmabuf.c @@ -28,7 +28,7 @@ uverbs_dmabuf_map(struct dma_buf_attachment *attachment, struct ib_uverbs_dmabuf_file *priv = attachment->dmabuf->priv; struct sg_table *ret; - dma_resv_assert_held(priv->dmabuf->resv); + dma_resv_assert_held(attachment->dmabuf->resv); if (priv->revoked) return ERR_PTR(-ENODEV); @@ -49,7 +49,7 @@ static void uverbs_dmabuf_unmap(struct dma_buf_attachment *attachment, { struct ib_uverbs_dmabuf_file *priv = attachment->dmabuf->priv; - dma_resv_assert_held(priv->dmabuf->resv); + dma_resv_assert_held(attachment->dmabuf->resv); dma_buf_free_sgt(attachment, sgt, dir); kref_put(&priv->kref, ib_uverbs_dmabuf_done); } @@ -68,12 +68,31 @@ static void uverbs_dmabuf_release(struct dma_buf *dmabuf) struct ib_uverbs_dmabuf_file *priv = dmabuf->priv; /* - * This can only happen if the fput came from alloc_abort_fd_uobject() + * This can only happen if the fput came from the error unwind of + * UVERBS_METHOD_DMABUF_ALLOC. The abort cleans up the uobject. */ - if (!priv->uobj.context) + if (!priv) return; + /* + * uverbs_uobject_release() may drop the last reference to the uobject, + * keep it around until the dmabuf pointer below has been cleared. + */ + uverbs_uobject_get(&priv->uobj); uverbs_uobject_release(&priv->uobj); + + /* + * The dma_buf is freed once this function returns. If the uobject was + * not destroyed above, because the ufile is concurrently being cleaned + * up, it is still reachable through the ufile's uobjects list and the + * mmap entry's dmabufs list. Make sure those paths are not in the + * middle of using the dma_buf and cannot use it afterwards. + */ + mutex_lock(&priv->dmabuf_release_lock); + priv->dmabuf = NULL; + mutex_unlock(&priv->dmabuf_release_lock); + + uverbs_uobject_put(&priv->uobj); } static const struct dma_buf_ops uverbs_dmabuf_ops = { @@ -99,6 +118,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DMABUF_ALLOC)( off_t pg_off; int ret; + mutex_init(&uverbs_dmabuf->dmabuf_release_lock); + ret = uverbs_get_const(&pg_off, attrs, UVERBS_ATTR_ALLOC_DMABUF_PGOFF); if (ret) return ret; @@ -141,6 +162,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_DMABUF_ALLOC)( return 0; err_revoked: + /* Tell uverbs_dmabuf_release() that the uobject is being aborted */ + uverbs_dmabuf->dmabuf->priv = NULL; dma_buf_put(uverbs_dmabuf->dmabuf); err: rdma_user_mmap_entry_put(mmap_entry); @@ -162,24 +185,11 @@ static void uverbs_dmabuf_fd_destroy_uobj(struct ib_uobject *uobj, { struct ib_uverbs_dmabuf_file *uverbs_dmabuf = container_of(uobj, struct ib_uverbs_dmabuf_file, uobj); - bool wait_for_comp = false; mutex_lock(&uverbs_dmabuf->mmap_entry->dmabufs_lock); - dma_resv_lock(uverbs_dmabuf->dmabuf->resv, NULL); if (!uverbs_dmabuf->revoked) { - uverbs_dmabuf->revoked = true; list_del(&uverbs_dmabuf->dmabufs_elm); - dma_buf_invalidate_mappings(uverbs_dmabuf->dmabuf); - dma_resv_wait_timeout(uverbs_dmabuf->dmabuf->resv, - DMA_RESV_USAGE_BOOKKEEP, false, - MAX_SCHEDULE_TIMEOUT); - wait_for_comp = true; - } - dma_resv_unlock(uverbs_dmabuf->dmabuf->resv); - if (wait_for_comp) { - kref_put(&uverbs_dmabuf->kref, ib_uverbs_dmabuf_done); - /* Let's wait till all DMA unmap are completed. */ - wait_for_completion(&uverbs_dmabuf->comp); + ib_uverbs_dmabuf_revoke(uverbs_dmabuf); } mutex_unlock(&uverbs_dmabuf->mmap_entry->dmabufs_lock); diff --git a/drivers/infiniband/core/uverbs_std_types_wq.c b/drivers/infiniband/core/uverbs_std_types_wq.c index 7ded8339346f..c23f455231bf 100644 --- a/drivers/infiniband/core/uverbs_std_types_wq.c +++ b/drivers/infiniband/core/uverbs_std_types_wq.c @@ -140,7 +140,7 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_WQ_CQ_HANDLE, UVERBS_OBJECT_CQ, UVERBS_ACCESS_READ, - UA_OPTIONAL), + UA_MANDATORY), UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_WQ_EVENT_FD, UVERBS_OBJECT_ASYNC_EVENT, UVERBS_ACCESS_READ, diff --git a/drivers/infiniband/core/uverbs_uapi.c b/drivers/infiniband/core/uverbs_uapi.c index d150099b99d2..b67db3790f03 100644 --- a/drivers/infiniband/core/uverbs_uapi.c +++ b/drivers/infiniband/core/uverbs_uapi.c @@ -729,7 +729,7 @@ void uverbs_disassociate_api(struct uverbs_api *uapi) * * release_cleanup is the exception because * uverbs_uobject_fd_release() needs it. In this case - * the module reference held by the fops will guarentee + * the module reference held by the fops will guarantee * the type_class remains valid too. */ if (type_attrs && diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index c43e25d37266..7d52d3036208 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1011,7 +1011,8 @@ int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr) ah->device->ops.modify_ah(ah, ah_attr) : -EOPNOTSUPP; - ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr); + if (!ret) + ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr); rdma_unfill_sgid_attr(ah_attr, old_sgid_attr); return ret; } @@ -1331,8 +1332,7 @@ static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd, return qp; err_security: - qp->device->ops.destroy_qp( - qp, uattrs ? uverbs_get_cleared_udata(uattrs) : NULL); + qp->device->ops.destroy_qp(qp, uverbs_get_cleared_udata(uattrs)); err_create: rdma_restrack_put(&qp->res); xa_destroy(&qp->comp_cntrs); @@ -2362,7 +2362,7 @@ EXPORT_SYMBOL(ib_dereg_mr_user); * @max_num_sg: maximum sg entries available for registration. * * Notes: - * Memory registeration page/sg lists must not exceed max_num_sg. + * Memory registration page/sg lists must not exceed max_num_sg. * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed * max_num_sg * used_page_size. * diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h index a43e678151d3..da471c63a3ba 100644 --- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h @@ -216,6 +216,10 @@ struct bnxt_re_dev { unsigned long event_bitmap; struct bnxt_qplib_cc_param cc_param; struct workqueue_struct *dcb_wq; + /* Protects dcb_wq against bnxt_re_uninit_dcb_wq() destroying it + * concurrently with bnxt_re_async_notifier() queuing work on it. + */ + spinlock_t dcb_lock; struct dentry *cc_config; struct bnxt_re_dbg_cc_config_params *cc_config_params; struct dentry *cq_coal_cfg; diff --git a/drivers/infiniband/hw/bnxt_re/hw_counters.c b/drivers/infiniband/hw/bnxt_re/hw_counters.c index 651cf9d0e0c7..575b97030587 100644 --- a/drivers/infiniband/hw/bnxt_re/hw_counters.c +++ b/drivers/infiniband/hw/bnxt_re/hw_counters.c @@ -413,7 +413,8 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, } done: - return bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ? + return bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn) ? BNXT_RE_NUM_EXT_COUNTERS : BNXT_RE_NUM_STD_COUNTERS; } @@ -423,7 +424,8 @@ struct rdma_hw_stats *bnxt_re_ib_alloc_hw_port_stats(struct ib_device *ibdev, struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); int num_counters = 0; - if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn)) num_counters = BNXT_RE_NUM_EXT_COUNTERS; else num_counters = BNXT_RE_NUM_STD_COUNTERS; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index ccd2702db78b..ac15372376a2 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -165,10 +165,11 @@ static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev, qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO; } -static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list, +static u32 bnxt_re_build_sgl(struct ib_sge *ib_sg_list, struct bnxt_qplib_sge *sg_list, int num) { - int i, total = 0; + u32 total = 0; + int i; for (i = 0; i < num; i++) { sg_list[i].addr = ib_sg_list[i].addr; @@ -2235,6 +2236,14 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, goto exit; } + if (srq_init_attr->attr.max_sge > dev_attr->max_srq_sges) { + ibdev_err(&rdev->ibdev, + "Create SRQ failed - max_sge %d exceeds supported %d", + srq_init_attr->attr.max_sge, dev_attr->max_srq_sges); + rc = -EINVAL; + goto exit; + } + if (srq_init_attr->srq_type != IB_SRQT_BASIC) { rc = -EOPNOTSUPP; goto exit; @@ -2388,6 +2397,7 @@ int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, spin_lock_irqsave(&srq->lock, flags); while (wr) { /* Transcribe each ib_recv_wr to qplib_swqe */ + wqe.flags = 0; wqe.num_sge = wr->num_sge; bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge); wqe.wr_id = wr->wr_id; @@ -2903,7 +2913,7 @@ static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp, qp->send_psn &= BTH_PSN_MASK; qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); /* DETH */ - /* Use the priviledged Q_Key for QP1 */ + /* Use the privileged Q_Key for QP1 */ qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); qp->qp1_hdr.deth.source_qpn = IB_QP1; @@ -3168,8 +3178,9 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, wr->sg_list[i].addr; sge_len = wr->sg_list[i].length; - if ((sge_len + wqe->inline_len) > - BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) { + if (sge_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH || + ((sge_len + wqe->inline_len) > + BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH)) { ibdev_err(&rdev->ibdev, "Inline data size requested > supported value"); return -EINVAL; @@ -3185,17 +3196,22 @@ static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev, static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev, const struct ib_send_wr *wr, - struct bnxt_qplib_swqe *wqe) + struct bnxt_qplib_swqe *wqe, + u32 *payload_sz) { - int payload_sz = 0; + int rc; - if (wr->send_flags & IB_SEND_INLINE) - payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe); - else - payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, - wqe->num_sge); + if (wr->send_flags & IB_SEND_INLINE) { + rc = bnxt_re_copy_inline_data(rdev, wr, wqe); + if (rc < 0) + return rc; + *payload_sz = rc; + } else { + *payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list, + wqe->num_sge); + } - return payload_sz; + return 0; } static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) @@ -3218,7 +3234,8 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp, const struct ib_send_wr *wr) { - int rc = 0, payload_sz = 0; + int rc = 0; + u32 payload_sz = 0; unsigned long flags; spin_lock_irqsave(&qp->sq_lock, flags); @@ -3234,11 +3251,9 @@ static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, goto bad; } - payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); - if (payload_sz < 0) { - rc = -EINVAL; + rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz); + if (rc) goto bad; - } wqe.wr_id = wr->wr_id; wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; @@ -3279,7 +3294,8 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, { struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp); struct bnxt_qplib_swqe wqe; - int rc = 0, payload_sz = 0; + int rc = 0; + u32 payload_sz = 0; unsigned long flags; spin_lock_irqsave(&qp->sq_lock, flags); @@ -3296,11 +3312,9 @@ int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, goto bad; } - payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe); - if (payload_sz < 0) { - rc = -EINVAL; + rc = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe, &payload_sz); + if (rc) goto bad; - } wqe.wr_id = wr->wr_id; switch (wr->opcode) { @@ -3749,12 +3763,13 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, unsigned int cqe, if (rc) goto fail; - cq->resize_umem = ib_umem_get_va(&rdev->ibdev, req.cq_va, - entries * sizeof(struct cq_base), - IB_ACCESS_LOCAL_WRITE); + cq->resize_umem = ib_umem_get_cq_buf_or_va(&rdev->ibdev, NULL, + req.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->resize_umem)) { rc = PTR_ERR(cq->resize_umem); - ibdev_err(&rdev->ibdev, "%s: ib_umem_get_va failed! rc = %pe\n", + ibdev_err(&rdev->ibdev, "%s: ib_umem_get_cq_buf_or_va failed! rc = %pe\n", __func__, cq->resize_umem); cq->resize_umem = NULL; goto fail; diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 17654a9e23fe..fa0e1323a326 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -367,9 +367,15 @@ static int bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) { - if (!rdev->dcb_wq) - return; - destroy_workqueue(rdev->dcb_wq); + struct workqueue_struct *dcb_wq; + + spin_lock_bh(&rdev->dcb_lock); + dcb_wq = rdev->dcb_wq; + rdev->dcb_wq = NULL; + spin_unlock_bh(&rdev->dcb_lock); + + if (dcb_wq) + destroy_workqueue(dcb_wq); } static void bnxt_re_dcb_wq_task(struct work_struct *work) @@ -424,14 +430,23 @@ static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *c switch (event_id) { case ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: - dcb_work = kzalloc_obj(*dcb_work, GFP_ATOMIC); - if (!dcb_work) + spin_lock(&rdev->dcb_lock); + if (!rdev->dcb_wq) { + spin_unlock(&rdev->dcb_lock); break; + } + + dcb_work = kzalloc_obj(*dcb_work, GFP_ATOMIC); + if (!dcb_work) { + spin_unlock(&rdev->dcb_lock); + break; + } dcb_work->rdev = rdev; memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl)); INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task); queue_work(rdev->dcb_wq, &dcb_work->work); + spin_unlock(&rdev->dcb_lock); break; default: break; @@ -1448,6 +1463,7 @@ static struct bnxt_re_dev *bnxt_re_dev_add(struct auxiliary_device *adev, INIT_LIST_HEAD(&rdev->qp_list); mutex_init(&rdev->qp_lock); mutex_init(&rdev->pacing.dbq_lock); + spin_lock_init(&rdev->dcb_lock); atomic_set(&rdev->stats.res.qp_count, 0); atomic_set(&rdev->stats.res.cq_count, 0); atomic_set(&rdev->stats.res.srq_count, 0); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c index 9ee0d500436c..34d8aa12265d 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_rcfw.c @@ -638,6 +638,10 @@ static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, err_event = (struct creq_qp_error_notification *)qp_event; qp_id = le32_to_cpu(err_event->xid); spin_lock(&rcfw->tbl_lock); + if (!rcfw->qp_tbl) { + spin_unlock(&rcfw->tbl_lock); + break; + } tbl_indx = map_qp_id_to_tbl_indx(qp_id, rcfw); qp = rcfw->qp_tbl[tbl_indx].qp_handle; if (!qp) { diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index 756f8b5f042a..cde56ff2313b 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -668,9 +669,9 @@ static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, { u32 bytes; - bytes = max >> 3; + bytes = BITS_TO_LONGS(max) * sizeof(unsigned long); if (!bytes) - bytes = 1; + bytes = sizeof(unsigned long); pdt->tbl = kmalloc(bytes, GFP_KERNEL); if (!pdt->tbl) return -ENOMEM; @@ -848,9 +849,9 @@ static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, if (!dpit->app_tbl) return -ENOMEM; - bytes = dpit->max >> 3; + bytes = BITS_TO_LONGS(dpit->max) * sizeof(unsigned long); if (!bytes) - bytes = 1; + bytes = sizeof(unsigned long); dpit->tbl = kmalloc(bytes, GFP_KERNEL); if (!dpit->tbl) { @@ -909,7 +910,15 @@ int bnxt_qplib_init_res(struct bnxt_qplib_res *res) void bnxt_qplib_free_res(struct bnxt_qplib_res *res) { - kfree(res->rcfw->qp_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_qp_node *qp_tbl; + + /* The CREQ tasklet may still be running; unpublish the table first */ + spin_lock_bh(&rcfw->tbl_lock); + qp_tbl = rcfw->qp_tbl; + rcfw->qp_tbl = NULL; + spin_unlock_bh(&rcfw->tbl_lock); + kvfree(qp_tbl); bnxt_qplib_free_sgid_tbl(res, &res->sgid_tbl); bnxt_qplib_free_pd_tbl(&res->pd_tbl); bnxt_qplib_free_dpi_tbl(res, &res->dpi_tbl); @@ -926,8 +935,8 @@ int bnxt_qplib_alloc_res(struct bnxt_qplib_res *res, struct net_device *netdev) /* Allocate one extra to hold the QP1 entries */ rcfw->qp_tbl_size = max_t(u32, BNXT_RE_MAX_QPC_COUNT + 1, dev_attr->max_qp); - rcfw->qp_tbl = kzalloc_objs(struct bnxt_qplib_qp_node, - rcfw->qp_tbl_size); + rcfw->qp_tbl = kvzalloc_objs(struct bnxt_qplib_qp_node, + rcfw->qp_tbl_size); if (!rcfw->qp_tbl) return -ENOMEM; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c index ec9eb52a8ebf..9aaa2b5204b8 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c @@ -160,7 +160,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) attr->max_srq = le16_to_cpu(sb->max_srq); attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; - attr->max_srq_sges = sb->max_srq_sge; + attr->max_srq_sges = min_t(u32, sb->max_srq_sge, BNXT_STATIC_MAX_SGE); attr->max_pkey = 1; attr->max_inline_data = attr->max_qp_sges * sizeof(struct sq_sge); if (!bnxt_qplib_is_chip_gen_p7(rcfw->res->cctx)) diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index b3b45c49077d..a6f520f49d59 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -4006,8 +4006,10 @@ static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb, int ret; req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL); - if (!req_skb) + if (!req_skb) { + kfree_skb(skb); return; + } req = __skb_put_zero(req_skb, sizeof(*req)); req->op_compl = htonl(WR_OP_V(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL_F); req->len16_pkd = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16))); @@ -4156,6 +4158,7 @@ static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb) pdev = ip_dev_find(&init_net, iph->daddr); if (!pdev) { pr_err("%s - failed to find device!\n", __func__); + neigh_release(neigh); goto free_dst; } e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index d1517f2560b9..2a7be0a1f168 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -1182,7 +1182,7 @@ void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx) struct c4iw_cq *rchp = to_c4iw_cq(qhp->ibqp.recv_cq); unsigned long flag; - /* locking heirarchy: cq lock first, then qp lock. */ + /* locking hierarchy: cq lock first, then qp lock. */ spin_lock_irqsave(&rchp->lock, flag); spin_lock(&qhp->lock); diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index ebe3170a641c..904a5323a693 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -171,6 +171,11 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) len, t4_pgprot_wc(vma->vm_page_prot)); break; case CXGB4_MMAP_CONTIG: + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) { + ret = -EPERM; + break; + } + vm_flags_clear(vma, VM_MAYWRITE); ret = io_remap_pfn_range(vma, vma->vm_start, addr >> PAGE_SHIFT, len, vma->vm_page_prot); diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h index ab830764e3b4..30a38e633bfd 100644 --- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h +++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h @@ -358,7 +358,8 @@ struct efa_admin_reg_mr_cmd { * 5:0 : phys_page_size_shift - page size is (1 << * phys_page_size_shift). Page size is used for * building the Virtual to Physical address mapping - * 6 : reserved - MBZ + * 6 : relaxed_ordering_enable - Enable PCIe relaxed + * ordering for this memory region * 7 : mem_addr_phy_mode_en - Enable bit for physical * memory registration (no translation), can be used * only by privileged clients. If set, PBL must @@ -1203,6 +1204,7 @@ struct efa_admin_modify_event_counter_resp { /* reg_mr_cmd */ #define EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT_MASK GENMASK(5, 0) +#define EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE_MASK BIT(6) #define EFA_ADMIN_REG_MR_CMD_MEM_ADDR_PHY_MODE_EN_MASK BIT(7) #define EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE_MASK BIT(0) #define EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE_MASK BIT(1) @@ -1231,7 +1233,6 @@ struct efa_admin_modify_event_counter_resp { #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK BIT(3) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4) #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_EVENT_COUNTERS_MASK BIT(5) -#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_SHIFT 10 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_SQ_64_BIT_REQ_ID_MASK BIT(10) /* feature_queue_attr_desc_2 */ diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index 04c6d63c449e..9c59d127e1e0 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -41,6 +41,7 @@ enum efa_cmd_status { EFA_CMD_ALLOCATED, EFA_CMD_SUBMITTED, EFA_CMD_COMPLETED, + EFA_CMD_TIMED_OUT, }; struct efa_comp_ctx { @@ -591,6 +592,7 @@ static int efa_com_comp_status_to_errno(u8 comp_status) static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx, struct efa_com_admin_queue *aq) { + bool comp_timeout = false; unsigned long timeout; unsigned long flags; @@ -599,12 +601,19 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c while (1) { spin_lock_irqsave(&aq->cq.lock, flags); efa_com_handle_admin_completion(aq); - spin_unlock_irqrestore(&aq->cq.lock, flags); - - if (comp_ctx->status != EFA_CMD_SUBMITTED) + if (comp_ctx->status != EFA_CMD_SUBMITTED) { + spin_unlock_irqrestore(&aq->cq.lock, flags); break; + } if (time_is_before_jiffies(timeout)) { + comp_ctx->status = EFA_CMD_TIMED_OUT; + comp_timeout = true; + } + + spin_unlock_irqrestore(&aq->cq.lock, flags); + + if (comp_timeout) { ibdev_err_ratelimited( aq->efa_dev, "Wait for completion (polling) timeout\n"); @@ -625,10 +634,11 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx, struct efa_com_admin_queue *aq) { - unsigned long flags; + unsigned long flags, remaining_jiffies; + bool completed; - wait_for_completion_timeout(&comp_ctx->wait_event, - usecs_to_jiffies(aq->completion_timeout)); + remaining_jiffies = wait_for_completion_timeout( + &comp_ctx->wait_event, usecs_to_jiffies(aq->completion_timeout)); /* * In case the command wasn't completed find out the root cause. @@ -636,14 +646,16 @@ static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *com * 1) No completion (timeout reached) * 2) There is completion but the device didn't get any msi-x interrupt. */ - if (comp_ctx->status == EFA_CMD_SUBMITTED) { + if (remaining_jiffies == 0) { spin_lock_irqsave(&aq->cq.lock, flags); efa_com_handle_admin_completion(aq); + completed = (comp_ctx->status == EFA_CMD_COMPLETED); + comp_ctx->status = EFA_CMD_TIMED_OUT; spin_unlock_irqrestore(&aq->cq.lock, flags); atomic64_inc(&aq->stats.no_completion); - if (comp_ctx->status == EFA_CMD_COMPLETED) + if (completed) ibdev_err_ratelimited( aq->efa_dev, "The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (id: %d, sq producer: %d, sq consumer: %d, cq consumer: %d)\n", diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index 1b00f16b8ea8..07bfcb093df8 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -236,8 +236,15 @@ int efa_com_register_mr(struct efa_com_dev *edev, mr_cmd.mr_length = params->mr_length_in_bytes; EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_PHYS_PAGE_SIZE_SHIFT, params->page_shift); + if (params->relaxed_ordering) + EFA_SET(&mr_cmd.flags, EFA_ADMIN_REG_MR_CMD_RELAXED_ORDERING_ENABLE, 1); mr_cmd.iova = params->iova; - mr_cmd.permissions = params->permissions; + if (params->permissions.local_write) + EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_LOCAL_WRITE_ENABLE, 1); + if (params->permissions.remote_write) + EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_WRITE_ENABLE, 1); + if (params->permissions.remote_read) + EFA_SET(&mr_cmd.permissions, EFA_ADMIN_REG_MR_CMD_REMOTE_READ_ENABLE, 1); if (params->inline_pbl) { memcpy(mr_cmd.pbl.inline_pbl_array, diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.h b/drivers/infiniband/hw/efa/efa_com_cmd.h index bca722f021ea..524407f36d43 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.h +++ b/drivers/infiniband/hw/efa/efa_com_cmd.h @@ -204,8 +204,12 @@ struct efa_com_reg_mr_params { * address mapping */ u8 page_shift; - /* see permissions field of struct efa_admin_reg_mr_cmd */ - u8 permissions; + struct { + u8 local_write : 1; + u8 remote_write : 1; + u8 remote_read : 1; + } permissions; + u8 relaxed_ordering; u8 inline_pbl; u8 indirect; }; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 2d28d68efe77..d56eb132545b 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -190,7 +190,7 @@ static void *efa_zalloc_mapped(struct efa_dev *dev, dma_addr_t *dma_addr, { void *addr; - addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO); + addr = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL); if (!addr) return NULL; @@ -1676,7 +1676,10 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start, params.pd = to_epd(ibpd)->pdn; params.iova = virt_addr; params.mr_length_in_bytes = length; - params.permissions = access_flags; + params.permissions.local_write = !!(access_flags & IB_ACCESS_LOCAL_WRITE); + params.permissions.remote_write = !!(access_flags & IB_ACCESS_REMOTE_WRITE); + params.permissions.remote_read = !!(access_flags & IB_ACCESS_REMOTE_READ); + params.relaxed_ordering = !!(access_flags & IB_ACCESS_RELAXED_ORDERING); pg_sz = ib_umem_find_best_pgsz(mr->umem, dev->dev_attr.page_size_cap, diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c index 1f456327e63c..daec74f0aa6d 100644 --- a/drivers/infiniband/hw/erdma/erdma_cq.c +++ b/drivers/infiniband/hw/erdma/erdma_cq.c @@ -8,8 +8,8 @@ static void *get_next_valid_cqe(struct erdma_cq *cq) { - __be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci, - cq->depth, CQE_SHIFT); + __be32 *cqe = erdma_kmem_get_entry( + &cq->kern_cq.qbuf_mem, cq->kern_cq.ci, cq->depth, CQE_SHIFT); u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK, be32_to_cpu(READ_ONCE(*cqe))); @@ -161,8 +161,8 @@ static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc) if (qtype == ERDMA_CQE_QTYPE_SQ) { id_table = kern_qp->swr_tbl; depth = qp->attrs.sq_size; - wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, - qp->attrs.sq_size, SQEBB_SHIFT); + wqe_hdr = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx, + qp->attrs.sq_size, SQEBB_SHIFT); kern_qp->sq_ci = FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) + wqe_idx + 1; @@ -242,15 +242,15 @@ void erdma_remove_cqes_of_qp(struct ib_cq *ibcq, u32 qpn) while (ncqe > 0) { cur_cq_ci = prev_cq_ci + ncqe - 1; - cqe = get_queue_entry(cq->kern_cq.qbuf, cur_cq_ci, cq->depth, - CQE_SHIFT); + cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem, cur_cq_ci, + cq->depth, CQE_SHIFT); if (be32_to_cpu(cqe->qpn) == qpn) { ++nqp_cqe; } else if (nqp_cqe) { - dst_cqe = get_queue_entry(cq->kern_cq.qbuf, - cur_cq_ci + nqp_cqe, - cq->depth, CQE_SHIFT); + dst_cqe = erdma_kmem_get_entry(&cq->kern_cq.qbuf_mem, + cur_cq_ci + nqp_cqe, + cq->depth, CQE_SHIFT); owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK, be32_to_cpu(dst_cqe->hdr)); cqe->hdr = cpu_to_be32( diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c index 445182c6bc5d..62dc22388eea 100644 --- a/drivers/infiniband/hw/erdma/erdma_main.c +++ b/drivers/infiniband/hw/erdma/erdma_main.c @@ -50,7 +50,7 @@ static int erdma_enum_and_get_netdev(struct erdma_dev *dev) struct net_device *netdev; int ret = -EPROBE_DEFER; - /* Already binded to a net_device, so we skip. */ + /* Already bound to a net_device, so we skip. */ if (dev->netdev) return 0; diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c index e002343832f7..fc335b674746 100644 --- a/drivers/infiniband/hw/erdma/erdma_qp.c +++ b/drivers/infiniband/hw/erdma/erdma_qp.c @@ -259,8 +259,8 @@ static void erdma_reset_qp(struct erdma_qp *qp) qp->kern_qp.rq_ci = 0; memset(qp->kern_qp.swr_tbl, 0, qp->attrs.sq_size * sizeof(u64)); memset(qp->kern_qp.rwr_tbl, 0, qp->attrs.rq_size * sizeof(u64)); - memset(qp->kern_qp.sq_buf, 0, qp->attrs.sq_size << SQEBB_SHIFT); - memset(qp->kern_qp.rq_buf, 0, qp->attrs.rq_size << RQE_SHIFT); + erdma_kmem_clear(&qp->kern_qp.sq_mem); + erdma_kmem_clear(&qp->kern_qp.rq_mem); erdma_remove_cqes_of_qp(&qp->scq->ibcq, QP_ID(qp)); if (qp->rcq != qp->scq) erdma_remove_cqes_of_qp(&qp->rcq->ibcq, QP_ID(qp)); @@ -332,8 +332,8 @@ static int fill_inline_data(struct erdma_qp *qp, wqe_idx += (sgl_offset >> SQEBB_SHIFT); sgl_offset &= (SQEBB_SIZE - 1); - data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size, - SQEBB_SHIFT); + data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx, + qp->attrs.sq_size, SQEBB_SHIFT); while (i < send_wr->num_sge) { bytes += send_wr->sg_list[i].length; @@ -356,8 +356,9 @@ static int fill_inline_data(struct erdma_qp *qp, wqe_idx += (sgl_offset >> SQEBB_SHIFT); sgl_offset &= (SQEBB_SIZE - 1); - data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, - qp->attrs.sq_size, SQEBB_SHIFT); + data = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, + wqe_idx, qp->attrs.sq_size, + SQEBB_SHIFT); if (!remain_size) break; } @@ -385,8 +386,8 @@ static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr, while (i < send_wr->num_sge) { wqe_idx += (sgl_offset >> SQEBB_SHIFT); sgl_offset &= (SQEBB_SIZE - 1); - sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, - qp->attrs.sq_size, SQEBB_SHIFT); + sgl = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, wqe_idx, + qp->attrs.sq_size, SQEBB_SHIFT); bytes += send_wr->sg_list[i].length; memcpy(sgl + sgl_offset, &send_wr->sg_list[i], @@ -463,8 +464,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, send_wr->opcode != IB_WR_SEND_WITH_IMM) return -EINVAL; - entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size, - SQEBB_SHIFT); + entry = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx, + qp->attrs.sq_size, SQEBB_SHIFT); /* Clear the SQE header section. */ *entry = 0; @@ -524,8 +525,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, read_sqe->sink_to_h = cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr)); - sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1, - qp->attrs.sq_size, SQEBB_SHIFT); + sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1, + qp->attrs.sq_size, SQEBB_SHIFT); sge->addr = cpu_to_le64(rdma_wr->remote_addr); sge->key = cpu_to_le32(rdma_wr->rkey); sge->length = cpu_to_le32(send_wr->sg_list[0].length); @@ -569,8 +570,9 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) { attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0); /* Copy SGLs to SQE content to accelerate */ - memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1, - qp->attrs.sq_size, SQEBB_SHIFT), + memcpy(erdma_kmem_get_entry(&qp->kern_qp.sq_mem, + idx + 1, qp->attrs.sq_size, + SQEBB_SHIFT), mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents)); wqe_size = sizeof(struct erdma_reg_mr_sqe) + MTT_SIZE(mr->mem.mtt_nents); @@ -605,8 +607,8 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi, cpu_to_le64(atomic_wr(send_wr)->compare_add); } - sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1, - qp->attrs.sq_size, SQEBB_SHIFT); + sge = erdma_kmem_get_entry(&qp->kern_qp.sq_mem, idx + 1, + qp->attrs.sq_size, SQEBB_SHIFT); sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr); sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey); sge++; @@ -702,8 +704,8 @@ static int erdma_post_recv_one(struct erdma_qp *qp, const struct ib_recv_wr *recv_wr) { struct erdma_rqe *rqe = - get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi, - qp->attrs.rq_size, RQE_SHIFT); + erdma_kmem_get_entry(&qp->kern_qp.rq_mem, qp->kern_qp.rq_pi, + qp->attrs.rq_size, RQE_SHIFT); rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1); rqe->qpn = cpu_to_le32(QP_ID(qp)); diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index f18b88bba281..dd29d55d4122 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -41,7 +41,7 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) struct erdma_dev *dev = to_edev(qp->ibqp.device); struct erdma_pd *pd = to_epd(qp->ibqp.pd); struct erdma_cmdq_create_qp_req req; - struct erdma_uqp *user_qp; + struct erdma_mem *sq_mem, *rq_mem; u64 resp0, resp1; int err; @@ -63,67 +63,46 @@ static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) ERDMA_QPT_UD); if (rdma_is_kernel_res(&qp->ibqp.res)) { - u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT; - - req.sq_cqn_mtt_cfg = - FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, - pgsz_range) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); - req.rq_cqn_mtt_cfg = - FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, - pgsz_range) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); - - req.sq_mtt_cfg = - FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK, - ERDMA_MR_MTT_0LEVEL); - req.rq_mtt_cfg = req.sq_mtt_cfg; - - req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; - req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; + sq_mem = &qp->kern_qp.sq_mem; + rq_mem = &qp->kern_qp.rq_mem; req.sq_dbrec_dma = qp->kern_qp.sq_dbrec_dma; req.rq_dbrec_dma = qp->kern_qp.rq_dbrec_dma; } else { - user_qp = &qp->user_qp; - req.sq_cqn_mtt_cfg = FIELD_PREP( - ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, - ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT); + sq_mem = &qp->user_qp.sq_mem; + rq_mem = &qp->user_qp.rq_mem; + req.sq_dbrec_dma = qp->user_qp.sq_dbrec_dma; + req.rq_dbrec_dma = qp->user_qp.rq_dbrec_dma; + } + + req.sq_cqn_mtt_cfg = + FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, + ilog2(sq_mem->page_size) - ERDMA_HW_PAGE_SHIFT); + req.sq_cqn_mtt_cfg |= + FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); + req.rq_cqn_mtt_cfg = + FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, + ilog2(rq_mem->page_size) - ERDMA_HW_PAGE_SHIFT); + req.rq_cqn_mtt_cfg |= + FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); + + req.sq_mtt_cfg = + sq_mem->page_offset | + FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, sq_mem->mtt_nents); + req.rq_mtt_cfg = + rq_mem->page_offset | + FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, rq_mem->mtt_nents); + + assemble_qbuf_mtt_for_cmd(sq_mem, &req.sq_mtt_cfg, &req.sq_buf_addr, + req.sq_mtt_entry); + assemble_qbuf_mtt_for_cmd(rq_mem, &req.rq_mtt_cfg, &req.rq_buf_addr, + req.rq_mtt_entry); + if (uctx && uctx->ext_db.enable) { req.sq_cqn_mtt_cfg |= - FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); - - req.rq_cqn_mtt_cfg = FIELD_PREP( - ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, - ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT); - req.rq_cqn_mtt_cfg |= - FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); - - req.sq_mtt_cfg = user_qp->sq_mem.page_offset; - req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, - user_qp->sq_mem.mtt_nents); - - req.rq_mtt_cfg = user_qp->rq_mem.page_offset; - req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, - user_qp->rq_mem.mtt_nents); - - assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg, - &req.sq_buf_addr, req.sq_mtt_entry); - assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg, - &req.rq_buf_addr, req.rq_mtt_entry); - - req.sq_dbrec_dma = user_qp->sq_dbrec_dma; - req.rq_dbrec_dma = user_qp->rq_dbrec_dma; - - if (uctx->ext_db.enable) { - req.sq_cqn_mtt_cfg |= - FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1); - req.db_cfg = - FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK, - uctx->ext_db.sdb_off) | - FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK, - uctx->ext_db.rdb_off); - } + FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1); + req.db_cfg = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK, + uctx->ext_db.sdb_off) | + FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK, + uctx->ext_db.rdb_off); } err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, &resp1, @@ -194,7 +173,6 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) struct erdma_dev *dev = to_edev(cq->ibcq.device); struct erdma_cmdq_create_cq_req req; struct erdma_mem *mem; - u32 page_size; erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_CREATE_CQ); @@ -204,48 +182,34 @@ static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn); if (rdma_is_kernel_res(&cq->ibcq.res)) { - page_size = SZ_32M; - req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, - ilog2(page_size) - ERDMA_HW_PAGE_SHIFT); - req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr); - req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr); - - req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) | - FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, - ERDMA_MR_MTT_0LEVEL); - - req.first_page_offset = 0; + mem = &cq->kern_cq.qbuf_mem; req.cq_dbrec_dma = cq->kern_cq.dbrec_dma; } else { mem = &cq->user_cq.qbuf_mem; - req.cfg0 |= - FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, - ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT); - if (mem->mtt_nents == 1) { - req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]); - req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]); - req.cfg1 |= - FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, - ERDMA_MR_MTT_0LEVEL); - } else { - req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma); - req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma); - req.cfg1 |= - FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, - ERDMA_MR_MTT_1LEVEL); - } - req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, - mem->mtt_nents); - - req.first_page_offset = mem->page_offset; req.cq_dbrec_dma = cq->user_cq.dbrec_dma; + } - if (uctx->ext_db.enable) { - req.cfg1 |= FIELD_PREP( - ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1); - req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK, - uctx->ext_db.cdb_off); - } + req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, + ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT); + if (mem->mtt_nents == 1) { + req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]); + req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]); + req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, + ERDMA_MR_MTT_0LEVEL); + } else { + req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma); + req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma); + req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK, + ERDMA_MR_MTT_1LEVEL); + } + req.cfg1 |= + FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, mem->mtt_nents); + req.first_page_offset = mem->page_offset; + + if (uctx && uctx->ext_db.enable) { + req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1); + req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK, + uctx->ext_db.cdb_off); } return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, @@ -505,96 +469,28 @@ static int erdma_qp_validate_attr(struct erdma_dev *dev, return 0; } -static void free_kernel_qp(struct erdma_qp *qp) +static void erdma_init_mtt_leaf(struct erdma_mem *mem, struct erdma_mtt *mtt) { - struct erdma_dev *dev = qp->dev; - - vfree(qp->kern_qp.swr_tbl); - vfree(qp->kern_qp.rwr_tbl); - - if (qp->kern_qp.sq_buf) - dma_free_coherent(&dev->pdev->dev, - qp->attrs.sq_size << SQEBB_SHIFT, - qp->kern_qp.sq_buf, - qp->kern_qp.sq_buf_dma_addr); - - if (qp->kern_qp.sq_dbrec) - dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, - qp->kern_qp.sq_dbrec_dma); - - if (qp->kern_qp.rq_buf) - dma_free_coherent(&dev->pdev->dev, - qp->attrs.rq_size << RQE_SHIFT, - qp->kern_qp.rq_buf, - qp->kern_qp.rq_buf_dma_addr); - - if (qp->kern_qp.rq_dbrec) - dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, - qp->kern_qp.rq_dbrec_dma); -} - -static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, - struct ib_qp_init_attr *attrs) -{ - struct erdma_kqp *kqp = &qp->kern_qp; - int size; - - if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) - kqp->sig_all = 1; - - kqp->sq_pi = 0; - kqp->sq_ci = 0; - kqp->rq_pi = 0; - kqp->rq_ci = 0; - kqp->hw_sq_db = - dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); - kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; - - kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64)); - kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64)); - if (!kqp->swr_tbl || !kqp->rwr_tbl) - goto err_out; - - size = qp->attrs.sq_size << SQEBB_SHIFT; - kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, - &kqp->sq_buf_dma_addr, GFP_KERNEL); - if (!kqp->sq_buf) - goto err_out; - - kqp->sq_dbrec = - dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); - if (!kqp->sq_dbrec) - goto err_out; - - size = qp->attrs.rq_size << RQE_SHIFT; - kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, - &kqp->rq_buf_dma_addr, GFP_KERNEL); - if (!kqp->rq_buf) - goto err_out; - - kqp->rq_dbrec = - dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); - if (!kqp->rq_dbrec) - goto err_out; - - return 0; - -err_out: - free_kernel_qp(qp); - return -ENOMEM; -} - -static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem) -{ - struct erdma_mtt *mtt = mem->mtt; struct ib_block_iter biter; u32 idx = 0; + if (mem->type == ERDMA_UMEM) { + rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) + mtt->buf[idx++] = rdma_block_iter_dma_address(&biter); + } else { + for (; idx < mem->page_cnt; idx++) + mtt->buf[idx] = mem->kmem.buf_list[idx].dma_addr; + } +} + +static void erdma_init_bottom_mtt(struct erdma_mem *mem) +{ + struct erdma_mtt *mtt = mem->mtt; + while (mtt->low_level) mtt = mtt->low_level; - rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) - mtt->buf[idx++] = rdma_block_iter_dma_address(&biter); + erdma_init_mtt_leaf(mem, mtt); } static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev, @@ -826,54 +722,216 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt) } } -static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, - u64 start, u64 len, int access, u64 virt, - unsigned long req_page_size, bool force_continuous) +static void erdma_kmem_free(struct erdma_dev *dev, struct erdma_mem *mem) { - int ret = 0; + struct erdma_buf_list *buf_list = mem->kmem.buf_list; + u32 i; - mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access); - if (IS_ERR(mem->umem)) { - ret = PTR_ERR(mem->umem); - mem->umem = NULL; - return ret; + for (i = 0; i < mem->page_cnt; i++) + dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf, + buf_list[i].dma_addr); + + kfree(buf_list); +} + +static int erdma_kmem_alloc(struct erdma_dev *dev, struct erdma_mem *mem, + u32 page_cnt) +{ + struct erdma_buf_list *buf_list; + u32 i; + + buf_list = kcalloc(page_cnt, sizeof(*buf_list), GFP_KERNEL); + if (!buf_list) + return -ENOMEM; + + for (i = 0; i < page_cnt; i++) { + buf_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, + &buf_list[i].dma_addr, + GFP_KERNEL); + if (!buf_list[i].buf) + goto err_free_pages; } - mem->va = virt; - mem->len = len; - mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt); - mem->page_offset = start & (mem->page_size - 1); - mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size); + mem->kmem.buf_list = buf_list; + return 0; + +err_free_pages: + while (i--) + dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf_list[i].buf, + buf_list[i].dma_addr); + kfree(buf_list); + + return -ENOMEM; +} + +static void erdma_mem_free(struct erdma_dev *dev, struct erdma_mem *mem) +{ + switch (mem->type) { + case ERDMA_UMEM: + ib_umem_release(mem->umem); + break; + case ERDMA_KMEM: + erdma_kmem_free(dev, mem); + break; + default: + break; + } +} + +static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem, + struct erdma_mem_init_attr *attr) +{ + struct erdma_mtt *mtt; + int ret; + + mem->type = attr->type; + + switch (mem->type) { + case ERDMA_UMEM: + if (attr->flags & ERDMA_MEM_FLAG_CQ_BUF) + mem->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, + attr->start, + attr->len, + attr->access); + else + mem->umem = ib_umem_get_va(&dev->ibdev, attr->start, + attr->len, attr->access); + if (IS_ERR(mem->umem)) { + ret = PTR_ERR(mem->umem); + return ret; + } + + mem->page_size = ib_umem_find_best_pgsz( + mem->umem, attr->req_page_size, attr->virt); + mem->mtt_nents = + ib_umem_num_dma_blocks(mem->umem, mem->page_size); + break; + case ERDMA_KMEM: + mem->page_size = PAGE_SIZE; + mem->mtt_nents = DIV_ROUND_UP(attr->len, PAGE_SIZE); + ret = erdma_kmem_alloc(dev, mem, mem->mtt_nents); + if (ret) + return ret; + break; + default: + return -EINVAL; + } + + mem->va = attr->virt; + mem->len = attr->len; + mem->page_offset = attr->start & (mem->page_size - 1); mem->page_cnt = mem->mtt_nents; - mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), - force_continuous); - if (IS_ERR(mem->mtt)) { - ret = PTR_ERR(mem->mtt); - goto error_ret; + mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt), + !(attr->flags & ERDMA_MEM_FLAG_MR_BUF)); + if (IS_ERR(mtt)) { + ret = PTR_ERR(mtt); + goto err_free_mem; } + mem->mtt = mtt; - erdma_fill_bottom_mtt(dev, mem); + erdma_init_bottom_mtt(mem); + if (mtt->continuous) + dma_sync_single_for_device(&dev->pdev->dev, mtt->buf_dma, + mtt->size, DMA_TO_DEVICE); return 0; -error_ret: - if (mem->umem) { - ib_umem_release(mem->umem); - mem->umem = NULL; - } +err_free_mem: + erdma_mem_free(dev, mem); return ret; } -static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem) +static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem) { - if (mem->mtt) - erdma_destroy_mtt(dev, mem->mtt); + erdma_destroy_mtt(dev, mem->mtt); + erdma_mem_free(dev, mem); +} - if (mem->umem) { - ib_umem_release(mem->umem); - mem->umem = NULL; +static void free_kernel_qp(struct erdma_qp *qp) +{ + struct erdma_dev *dev = qp->dev; + + vfree(qp->kern_qp.swr_tbl); + vfree(qp->kern_qp.rwr_tbl); + + erdma_mem_uninit(dev, &qp->kern_qp.sq_mem); + dma_pool_free(dev->db_pool, qp->kern_qp.sq_dbrec, + qp->kern_qp.sq_dbrec_dma); + erdma_mem_uninit(dev, &qp->kern_qp.rq_mem); + dma_pool_free(dev->db_pool, qp->kern_qp.rq_dbrec, + qp->kern_qp.rq_dbrec_dma); +} + +static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, + struct ib_qp_init_attr *attrs) +{ + struct erdma_mem_init_attr attr = { + .type = ERDMA_KMEM, + }; + struct erdma_kqp *kqp = &qp->kern_qp; + int ret = -ENOMEM; + int size; + + if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) + kqp->sig_all = 1; + + kqp->sq_pi = 0; + kqp->sq_ci = 0; + kqp->rq_pi = 0; + kqp->rq_ci = 0; + kqp->hw_sq_db = + dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); + kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; + + kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64)); + if (!kqp->swr_tbl) + return -ENOMEM; + + kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64)); + if (!kqp->rwr_tbl) + goto err_free_swr_tbl; + + size = qp->attrs.sq_size << SQEBB_SHIFT; + attr.len = size; + ret = erdma_mem_init(dev, &kqp->sq_mem, &attr); + if (ret) + goto err_free_rwr_tbl; + + kqp->sq_dbrec = + dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->sq_dbrec_dma); + if (!kqp->sq_dbrec) { + ret = -ENOMEM; + goto err_free_sq_mem; } + + size = qp->attrs.rq_size << RQE_SHIFT; + attr.len = size; + ret = erdma_mem_init(dev, &kqp->rq_mem, &attr); + if (ret) + goto err_free_sq_dbrec; + + kqp->rq_dbrec = + dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &kqp->rq_dbrec_dma); + if (!kqp->rq_dbrec) { + ret = -ENOMEM; + goto err_free_rq_mem; + } + + return 0; + +err_free_rq_mem: + erdma_mem_uninit(dev, &kqp->rq_mem); +err_free_sq_dbrec: + dma_pool_free(dev->db_pool, kqp->sq_dbrec, kqp->sq_dbrec_dma); +err_free_sq_mem: + erdma_mem_uninit(dev, &kqp->sq_mem); +err_free_rwr_tbl: + vfree(kqp->rwr_tbl); +err_free_swr_tbl: + vfree(kqp->swr_tbl); + + return ret; } static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, @@ -941,6 +999,9 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, u64 va, u32 len, u64 dbrec_va) { + struct erdma_mem_init_attr attr = { + .type = ERDMA_UMEM, + }; dma_addr_t dbrec_dma; u32 rq_offset; int ret; @@ -949,45 +1010,47 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, qp->attrs.rq_size * RQE_SIZE)) return -EINVAL; - ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va, - qp->attrs.sq_size << SQEBB_SHIFT, 0, va, - (SZ_1M - SZ_4K), true); + attr.start = va; + attr.virt = va; + attr.len = qp->attrs.sq_size << SQEBB_SHIFT; + attr.req_page_size = SZ_1M - SZ_4K; + ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, &attr); if (ret) return ret; rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE); qp->user_qp.rq_offset = rq_offset; - ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset, - qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset, - (SZ_1M - SZ_4K), true); + attr.start = va + rq_offset; + attr.virt = va + rq_offset; + attr.len = qp->attrs.rq_size << RQE_SHIFT; + ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, &attr); if (ret) - goto put_sq_mtt; + goto uninit_sq_mem; ret = erdma_map_user_dbrecords(uctx, dbrec_va, - &qp->user_qp.user_dbr_page, - &dbrec_dma); + &qp->user_qp.user_dbr_page, &dbrec_dma); if (ret) - goto put_rq_mtt; + goto uninit_rq_mem; qp->user_qp.sq_dbrec_dma = dbrec_dma; qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE; return 0; -put_rq_mtt: - put_mtt_entries(qp->dev, &qp->user_qp.rq_mem); +uninit_rq_mem: + erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem); -put_sq_mtt: - put_mtt_entries(qp->dev, &qp->user_qp.sq_mem); +uninit_sq_mem: + erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem); return ret; } static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx) { - put_mtt_entries(qp->dev, &qp->user_qp.sq_mem); - put_mtt_entries(qp->dev, &qp->user_qp.rq_mem); + erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem); + erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem); erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page); } @@ -1234,8 +1297,11 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, u64 virt, int access, struct ib_dmah *dmah, struct ib_udata *udata) { - struct erdma_mr *mr = NULL; struct erdma_dev *dev = to_edev(ibpd->device); + struct erdma_mem_init_attr attr = { + .type = ERDMA_UMEM, + }; + struct erdma_mr *mr = NULL; u32 stag; int ret; @@ -1249,8 +1315,13 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, if (!mr) return ERR_PTR(-ENOMEM); - ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt, - SZ_2G - SZ_4K, false); + attr.start = start; + attr.virt = virt; + attr.len = len; + attr.req_page_size = SZ_2G - SZ_4K; + attr.access = access; + attr.flags = ERDMA_MEM_FLAG_MR_BUF; + ret = erdma_mem_init(dev, &mr->mem, &attr); if (ret) goto err_out_free; @@ -1260,8 +1331,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr->ibmr.lkey = mr->ibmr.rkey = stag; mr->ibmr.pd = ibpd; - mr->mem.va = virt; - mr->mem.len = len; mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access); mr->valid = 1; mr->type = ERDMA_MR_TYPE_NORMAL; @@ -1277,7 +1346,7 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, mr->ibmr.lkey >> 8); err_out_put_mtt: - put_mtt_entries(dev, &mr->mem); + erdma_mem_uninit(dev, &mr->mem); err_out_free: kfree(mr); @@ -1314,7 +1383,8 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); - put_mtt_entries(dev, &mr->mem); + if (mr->type != ERDMA_MR_TYPE_DMA) + erdma_mem_uninit(dev, &mr->mem); kfree(mr); return 0; @@ -1349,13 +1419,12 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) wait_for_completion(&cq->free); if (rdma_is_kernel_res(&cq->ibcq.res)) { - dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, - cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem); dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, cq->kern_cq.dbrec_dma); } else { erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); - put_mtt_entries(dev, &cq->user_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem); } return 0; @@ -1407,8 +1476,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (rdma_is_kernel_res(&qp->ibqp.res)) { free_kernel_qp(qp); } else { - put_mtt_entries(dev, &qp->user_qp.sq_mem); - put_mtt_entries(dev, &qp->user_qp.rq_mem); + erdma_mem_uninit(dev, &qp->user_qp.sq_mem); + erdma_mem_uninit(dev, &qp->user_qp.rq_mem); erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page); } @@ -1926,12 +1995,19 @@ int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, struct erdma_ureq_create_cq *ureq) { - int ret; struct erdma_dev *dev = to_edev(cq->ibcq.device); + struct erdma_mem_init_attr attr = { + .type = ERDMA_UMEM, + }; + int ret; - ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va, - ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K, - true); + attr.start = ureq->qbuf_va; + attr.virt = ureq->qbuf_va; + attr.len = ureq->qbuf_len; + attr.req_page_size = SZ_64M - SZ_4K; + attr.access = IB_ACCESS_LOCAL_WRITE; + attr.flags = ERDMA_MEM_FLAG_CQ_BUF; + ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr); if (ret) return ret; @@ -1939,7 +2015,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, &cq->user_cq.user_dbr_page, &cq->user_cq.dbrec_dma); if (ret) - put_mtt_entries(dev, &cq->user_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem); return ret; } @@ -1947,12 +2023,15 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, static int erdma_init_kernel_cq(struct erdma_cq *cq) { struct erdma_dev *dev = to_edev(cq->ibcq.device); + struct erdma_mem_init_attr attr = { + .type = ERDMA_KMEM, + .len = cq->depth << CQE_SHIFT, + }; + int ret; - cq->kern_cq.qbuf = - dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, - &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); - if (!cq->kern_cq.qbuf) - return -ENOMEM; + ret = erdma_mem_init(dev, &cq->kern_cq.qbuf_mem, &attr); + if (ret) + return ret; cq->kern_cq.dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &cq->kern_cq.dbrec_dma); @@ -1966,8 +2045,7 @@ static int erdma_init_kernel_cq(struct erdma_cq *cq) return 0; err_out: - dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, - cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem); return -ENOMEM; } @@ -2032,10 +2110,9 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, err_free_res: if (!rdma_is_kernel_res(&ibcq->res)) { erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); - put_mtt_entries(dev, &cq->user_cq.qbuf_mem); + erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem); } else { - dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT, - cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); + erdma_mem_uninit(dev, &cq->kern_cq.qbuf_mem); dma_pool_free(dev->db_pool, cq->kern_cq.dbrec, cq->kern_cq.dbrec_dma); } diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index c73cecf92f61..0542e10a0c99 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -111,19 +111,71 @@ struct erdma_mtt { struct erdma_mtt *low_level; }; +enum erdma_mem_type { + ERDMA_NO_MEM = 0, + ERDMA_UMEM = 1, + ERDMA_KMEM = 2, +}; + +struct erdma_buf_list { + void *buf; + dma_addr_t dma_addr; +}; + +struct erdma_kmem { + struct erdma_buf_list *buf_list; +}; + +enum erdma_mem_flags { + ERDMA_MEM_FLAG_MR_BUF = (1 << 0), + ERDMA_MEM_FLAG_CQ_BUF = (1 << 1), +}; + +struct erdma_mem_init_attr { + enum erdma_mem_type type; + u64 start; + u64 virt; + u64 len; + unsigned long req_page_size; + int access; + u32 flags; +}; + struct erdma_mem { - struct ib_umem *umem; - struct erdma_mtt *mtt; + enum erdma_mem_type type; + union { + struct ib_umem *umem; + struct erdma_kmem kmem; + }; u32 page_size; u32 page_offset; u32 page_cnt; u32 mtt_nents; + struct erdma_mtt *mtt; + u64 va; u64 len; }; +static inline void *erdma_kmem_get_entry(struct erdma_mem *mem, u32 idx, + u32 depth, u32 shift) +{ + u32 offset = (idx & (depth - 1)) << shift; + + return (u8 *)mem->kmem.buf_list[offset >> PAGE_SHIFT].buf + + offset_in_page(offset); +} + +static inline void erdma_kmem_clear(struct erdma_mem *mem) +{ + u32 i; + + for (i = 0; i < mem->page_cnt; i++) + memset(mem->kmem.buf_list[i].buf, 0, PAGE_SIZE); +} + struct erdma_mr { struct ib_mr ibmr; struct erdma_mem mem; @@ -183,11 +235,8 @@ struct erdma_kqp { void __iomem *hw_sq_db; void __iomem *hw_rq_db; - void *sq_buf; - dma_addr_t sq_buf_dma_addr; - - void *rq_buf; - dma_addr_t rq_buf_dma_addr; + struct erdma_mem sq_mem; + struct erdma_mem rq_mem; void *sq_dbrec; void *rq_dbrec; @@ -320,8 +369,7 @@ struct erdma_qp { }; struct erdma_kcq_info { - void *qbuf; - dma_addr_t qbuf_dma_addr; + struct erdma_mem qbuf_mem; u32 ci; u32 cmdsn; u32 notify_cnt; diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c index 2b20907c9c12..d4bf75dcd102 100644 --- a/drivers/infiniband/hw/hfi1/affinity.c +++ b/drivers/infiniband/hw/hfi1/affinity.c @@ -386,7 +386,7 @@ static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd) /* * This function creates the table for looking up CPUs for completion vectors. - * num_comp_vectors needs to have been initilized before calling this function. + * num_comp_vectors needs to have been initialized before calling this function. */ static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd, struct hfi1_affinity_node *entry) diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 592e330e74bf..faa724ca33bd 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -5567,7 +5567,7 @@ static int init_rcverr(struct hfi1_devdata *dd) static void free_rcverr(struct hfi1_devdata *dd) { if (dd->rcverr_timer.function) - timer_delete_sync(&dd->rcverr_timer); + timer_shutdown_sync(&dd->rcverr_timer); } static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) @@ -12300,7 +12300,7 @@ static void free_cntrs(struct hfi1_devdata *dd) int i; if (dd->synth_stats_timer.function) - timer_delete_sync(&dd->synth_stats_timer); + timer_shutdown_sync(&dd->synth_stats_timer); cancel_work_sync(&dd->update_cntr_work); ppd = (struct hfi1_pportdata *)(dd + 1); for (i = 0; i < dd->num_pports; i++, ppd++) { diff --git a/drivers/infiniband/hw/hfi1/firmware.c b/drivers/infiniband/hw/hfi1/firmware.c index 3c228aeaaf81..a69de0a91238 100644 --- a/drivers/infiniband/hw/hfi1/firmware.c +++ b/drivers/infiniband/hw/hfi1/firmware.c @@ -77,7 +77,7 @@ struct css_header { /* size of platform configuration partition */ #define MAX_PLATFORM_CONFIG_FILE_SIZE 4096 -/* size of file of plaform configuration encoded in format version 4 */ +/* size of file of platform configuration encoded in format version 4 */ #define PLATFORM_CONFIG_FORMAT_4_FILE_SIZE 528 /* the file itself */ diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index 1154b8cc713c..106bed4932c8 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -53,17 +53,9 @@ int hfi1_pcie_init(struct pci_dev *pdev) ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (ret) { - /* - * If the 64 bit setup fails, try 32 bit. Some systems - * do not setup 64 bit maps on systems with 2GB or less - * memory installed. - */ - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (ret) { - dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n", - ERR_PTR(ret)); - goto bail; - } + dev_err(&pdev->dev, "Unable to set DMA mask: %pe\n", + ERR_PTR(ret)); + goto bail; } pci_set_master(pdev); diff --git a/drivers/infiniband/hw/hfi1/sdma.c b/drivers/infiniband/hw/hfi1/sdma.c index f253c8ee182d..a71fc8f2d37e 100644 --- a/drivers/infiniband/hw/hfi1/sdma.c +++ b/drivers/infiniband/hw/hfi1/sdma.c @@ -1569,6 +1569,9 @@ void sdma_exit(struct hfi1_devdata *dd) * it is not running. */ sdma_finalput(&sde->state); + cancel_work_sync(&sde->err_halt_worker); + tasklet_kill(&sde->sdma_hw_clean_up_task); + cancel_work_sync(&sde->flush_worker); } } diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index e569b647d611..0ef3b1a89a53 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1888,6 +1888,10 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) void hfi1_unregister_ib_device(struct hfi1_devdata *dd) { struct hfi1_ibdev *dev = &dd->verbs_dev; + int i; + + for (i = 0; i < dd->num_pports; i++) + timer_shutdown_sync(&dd->pport[i].ibport_data.rvp.trap_timer); hfi1_verbs_unregister_sysfs(dd); diff --git a/drivers/infiniband/hw/hns/hns_roce_bond.c b/drivers/infiniband/hw/hns/hns_roce_bond.c index f2b12ae13a58..8d8272705fb7 100644 --- a/drivers/infiniband/hw/hns/hns_roce_bond.c +++ b/drivers/infiniband/hw/hns/hns_roce_bond.c @@ -81,7 +81,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp, { struct net_device *active_dev; struct net_device *old_dev; - int i, ret = 0; + int i = 0, ret = 0; if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) { rcu_read_lock(); @@ -89,7 +89,7 @@ static int hns_roce_set_bond_netdev(struct hns_roce_bond_group *bond_grp, bond_option_active_slave_get_rcu(netdev_priv(bond_grp->upper_dev)); rcu_read_unlock(); } else { - for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) { + for (; i < ROCE_BOND_FUNC_MAX; i++) { active_dev = bond_grp->bond_func_info[i].net_dev; if (active_dev && ib_get_curr_port_state(active_dev) == IB_PORT_ACTIVE) diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 1dd0efb5620d..7dfaa01cda67 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -261,10 +261,11 @@ static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size; buf_attr.region[0].hopnum = hr_dev->caps.cqe_hop_num; buf_attr.region_count = 1; + buf_attr.user_access = IB_ACCESS_LOCAL_WRITE; ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr, hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT, - udata, addr); + udata, addr, true); if (ret) ibdev_err(ibdev, "failed to alloc CQ mtr, ret = %d.\n", ret); diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c index 103b8c9ca969..4ef0b882ea2f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c @@ -338,6 +338,60 @@ static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev, } } +static int gsi_sl_debugfs_show(struct seq_file *file, void *offset) +{ + struct hns_roce_dev *hr_dev = file->private; + + seq_printf(file, "%u\n", hr_dev->gsi_sl); + + return 0; +} + +static ssize_t gsi_sl_debugfs_store(char *buf, size_t count, void *data) +{ + struct hns_roce_dev *hr_dev = data; + struct net_device *netdev; + int ret; + u8 val; + + netdev = ib_device_get_netdev(&hr_dev->ib_dev, 1); + if (!netdev) + return -ENODEV; + + if (ib_get_curr_port_state(netdev) == IB_PORT_ACTIVE) { + ret = -EOPNOTSUPP; + goto out; + } + + ret = kstrtou8(buf, 0, &val); + if (ret) + goto out; + + if (!check_sl_valid(hr_dev, val)) { + ret = -EINVAL; + goto out; + } + + hr_dev->gsi_sl = val; + +out: + dev_put(netdev); + return ret ? : count; +} + +static void create_gsi_sl_debugfs(struct hns_roce_dev *hr_dev, + struct dentry *parent) +{ + struct hns_debugfs_seqfile *seqfile = &hr_dev->dbgfs.gsi_sl; + + seqfile->read = gsi_sl_debugfs_show; + seqfile->write = gsi_sl_debugfs_store; + seqfile->data = hr_dev; + + debugfs_create_file("gsi_sl", 0600, parent, seqfile, + &hns_debugfs_seqfile_fops); +} + /* debugfs for device */ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev) { @@ -348,6 +402,7 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev) create_sw_stat_debugfs(hr_dev, dbgfs->root); create_cc_param_debugfs(hr_dev, dbgfs->root); + create_gsi_sl_debugfs(hr_dev, dbgfs->root); } void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev) diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h index 116b1e8b6677..286704711785 100644 --- a/drivers/infiniband/hw/hns/hns_roce_debugfs.h +++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.h @@ -47,6 +47,7 @@ struct hns_roce_dev_debugfs { struct dentry *root; struct hns_sw_stat_debugfs sw_stat_root; struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM]; + struct hns_debugfs_seqfile gsi_sl; }; struct hns_roce_dev; diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index eb7b1865e4c7..5890ee69247c 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -654,6 +654,7 @@ struct hns_roce_qp { u8 priority; spinlock_t flush_lock; struct hns_roce_dip *dip; + bool ud_sl_set; }; struct hns_roce_ib_iboe { @@ -737,7 +738,7 @@ enum hns_roce_scc_algo { struct hns_roce_caps { u64 fw_ver; u8 num_ports; - int gid_table_len[HNS_ROCE_MAX_PORTS]; + u32 gid_table_len[HNS_ROCE_MAX_PORTS]; int pkey_table_len[HNS_ROCE_MAX_PORTS]; int local_ca_ack_delay; int num_uars; @@ -1047,6 +1048,7 @@ struct hns_roce_dev { struct hns_roce_dev_debugfs dbgfs; atomic64_t *dfx_cnt; struct hns_roce_scc_param *scc_param; + u8 gsi_sl; }; enum hns_roce_trace_type { @@ -1230,7 +1232,7 @@ int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, struct hns_roce_buf_attr *buf_attr, unsigned int page_shift, struct ib_udata *udata, - unsigned long user_addr); + unsigned long user_addr, bool is_cq); void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr); int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 27cc7df55ee7..a8f319a57f25 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -431,7 +431,8 @@ static int set_ud_opcode(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, return 0; } -static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, +static int fill_ud_av(struct hns_roce_qp *qp, + struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, struct hns_roce_ah *ah) { struct ib_device *ib_dev = ah->ibah.device; @@ -441,7 +442,13 @@ static int fill_ud_av(struct hns_roce_v2_ud_send_wqe *ud_sq_wqe, hr_reg_write(ud_sq_wqe, UD_SEND_WQE_HOPLIMIT, ah->av.hop_limit); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_TCLASS, ah->av.tclass); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_FLOW_LABEL, ah->av.flowlabel); - hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, ah->av.sl); + if (!qp->ud_sl_set || qp->ibqp.qp_type == IB_QPT_GSI) { + qp->sl = qp->ibqp.qp_type == IB_QPT_GSI ? + hr_dev->gsi_sl : ah->av.sl; + qp->ud_sl_set = true; + } + + hr_reg_write(ud_sq_wqe, UD_SEND_WQE_SL, qp->sl); ud_sq_wqe->sgid_index = ah->av.gid_index; @@ -491,12 +498,10 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp, qp->qkey : ud_wr(wr)->remote_qkey); hr_reg_write(ud_sq_wqe, UD_SEND_WQE_DQPN, ud_wr(wr)->remote_qpn); - ret = fill_ud_av(ud_sq_wqe, ah); + ret = fill_ud_av(qp, ud_sq_wqe, ah); if (ret) return ret; - qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl; - set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge); /* @@ -2416,8 +2421,7 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev) caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / caps->gmv_entry_sz)); - caps->gmv_entry_num = caps->gmv_bt_num * (HNS_HW_PAGE_SIZE / - caps->gmv_entry_sz); + caps->gmv_entry_num = caps->gid_table_len[0]; } else { u32 func_num = max_t(u32, 1, hr_dev->func_num); @@ -3824,7 +3828,8 @@ static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn, * Now backwards through the CQ, removing CQ entries * that match our QP by overwriting them with next entries. */ - while ((int) --prod_index - (int) hr_cq->cons_index >= 0) { + while (prod_index != hr_cq->cons_index) { + --prod_index; cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe); if (hr_reg_read(cqe, CQE_LCL_QPN) == qpn) { if (srq && hr_reg_read(cqe, CQE_S_R)) { @@ -4353,18 +4358,39 @@ static int get_op_for_set_hem(struct hns_roce_dev *hr_dev, u32 type, static int config_gmv_ba_to_hw(struct hns_roce_dev *hr_dev, unsigned long obj, dma_addr_t base_addr) { + u32 obj_num_per_bt = HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz; + u32 chunk_size = 1 << (hr_dev->caps.gmv_buf_pg_sz + PAGE_SHIFT); + u32 bt_num_per_chunk = chunk_size / HNS_HW_PAGE_SIZE; + u32 first = obj / obj_num_per_bt; + u32 last = min(first + bt_num_per_chunk, hr_dev->caps.gmv_bt_num); struct hns_roce_cmq_desc desc; - struct hns_roce_cmq_req *req = (struct hns_roce_cmq_req *)desc.data; - u32 idx = obj / (HNS_HW_PAGE_SIZE / hr_dev->caps.gmv_entry_sz); - u64 addr = to_hr_hw_page_addr(base_addr); + struct hns_roce_cmq_req *req; + u64 addr; + int ret; + u32 i; - hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, false); + /* The GMV BT entry of hardware covers a fixed 4K region, so a buffer + * chunk larger than 4K must be registered to hardware with one BT + * entry per 4K block, otherwise the GMV entries beyond the first + * 4K of the chunk are unreachable. + */ + for (i = first; i < last; i++) { + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GMV_BT, + false); + req = (struct hns_roce_cmq_req *)desc.data; - hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); - hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); - hr_reg_write(req, CFG_GMV_BT_IDX, idx); + addr = to_hr_hw_page_addr(base_addr + + (u64)(i - first) * HNS_HW_PAGE_SIZE); + hr_reg_write(req, CFG_GMV_BT_BA_L, lower_32_bits(addr)); + hr_reg_write(req, CFG_GMV_BT_BA_H, upper_32_bits(addr)); + hr_reg_write(req, CFG_GMV_BT_IDX, i); - return hns_roce_cmq_send(hr_dev, &desc, 1); + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + return ret; + } + + return 0; } static int set_hem_to_hw(struct hns_roce_dev *hr_dev, int obj, @@ -5327,7 +5353,7 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, hr_reg_clear(qpc_mask, QPC_VLAN_ID); if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) { - ibdev_err(ibdev, "sgid_index(%u) too large. max is %d\n", + ibdev_err(ibdev, "sgid_index(%u) too large. max is %u\n", grh->sgid_index, hr_dev->caps.gid_table_len[hr_port]); return -EINVAL; } @@ -5603,6 +5629,7 @@ static void v2_set_flushed_fields(struct ib_qp *ibqp, hr_reg_write(context, QPC_SQ_PRODUCER_IDX, hr_qp->sq.head); hr_reg_clear(qpc_mask, QPC_SQ_PRODUCER_IDX); hr_qp->state = IB_QPS_ERR; + hr_qp->ud_sl_set = false; spin_unlock_irqrestore(&hr_qp->sq.lock, sq_flag); if (ibqp->srq || ibqp->qp_type == IB_QPT_XRC_INI) /* no RQ */ @@ -6913,7 +6940,7 @@ static int alloc_eq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq) err = hns_roce_mtr_create(hr_dev, &eq->mtr, &buf_attr, hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT, NULL, - 0); + 0, false); if (err) dev_err(hr_dev->dev, "failed to alloc EQE mtr, err %d\n", err); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 7d41ae897458..4799f667eeda 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -112,7 +112,7 @@ static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr, hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT, - udata, start); + udata, start, false); if (err) { ibdev_err(ibdev, "failed to alloc pbl mtr, ret = %d.\n", err); return err; @@ -586,7 +586,8 @@ static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, struct hns_roce_buf_attr *buf_attr, - struct ib_udata *udata, unsigned long user_addr) + struct ib_udata *udata, unsigned long user_addr, + bool is_cq) { struct ib_device *ibdev = &hr_dev->ib_dev; size_t total_size; @@ -595,8 +596,14 @@ static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, if (udata) { mtr->kmem = NULL; - mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size, - buf_attr->user_access); + if (is_cq) + mtr->umem = ib_umem_get_cq_buf_or_va(ibdev, NULL, + user_addr, + total_size, + buf_attr->user_access); + else + mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size, + buf_attr->user_access); if (IS_ERR(mtr->umem)) { ibdev_err(ibdev, "failed to get umem, ret = %pe.\n", mtr->umem); @@ -1035,11 +1042,13 @@ static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr) * @ba_page_shift: page shift for multi-hop base address table * @udata: user space context, if it's NULL, means kernel space * @user_addr: userspace virtual address to start at + * @is_cq: true when @mtr backs a CQ buffer, which the device writes + * completion entries into */ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, struct hns_roce_buf_attr *buf_attr, unsigned int ba_page_shift, struct ib_udata *udata, - unsigned long user_addr) + unsigned long user_addr, bool is_cq) { struct ib_device *ibdev = &hr_dev->ib_dev; int ret; @@ -1052,7 +1061,8 @@ int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, mtr->umem = NULL; mtr->kmem = NULL; } else { - ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr); + ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr, + is_cq); if (ret) { ibdev_err(ibdev, "failed to alloc mtr bufs, ret = %d.\n", ret); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index e333a8c4acb5..c6c0dfdbdbc3 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -808,7 +808,7 @@ static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp, } ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr, PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz, - udata, addr); + udata, addr, false); if (ret) { ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret); goto err_inline; diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index 4a54394f96be..b5e9ac9cfd59 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -180,7 +180,7 @@ static int alloc_srq_idx(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq, ret = hns_roce_mtr_create(hr_dev, &idx_que->mtr, &buf_attr, hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT, - udata, addr); + udata, addr, false); if (ret) { ibdev_err(ibdev, "failed to alloc SRQ idx mtr, ret = %d.\n", ret); @@ -235,7 +235,7 @@ static int alloc_srq_wqe_buf(struct hns_roce_dev *hr_dev, ret = hns_roce_mtr_create(hr_dev, &srq->buf_mtr, &buf_attr, hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT, - udata, addr); + udata, addr, false); if (ret) ibdev_err(ibdev, "failed to alloc SRQ buf mtr, ret = %d.\n", ret); diff --git a/drivers/infiniband/hw/ionic/ionic_admin.c b/drivers/infiniband/hw/ionic/ionic_admin.c index 37e24450d129..0ccd869ad8e6 100644 --- a/drivers/infiniband/hw/ionic/ionic_admin.c +++ b/drivers/infiniband/hw/ionic/ionic_admin.c @@ -489,6 +489,49 @@ static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev, return ionic_rdma_devcmd(dev, &admin); } +static int ionic_rdma_cq_devcmd(struct ionic_vcq *vcq, + struct ionic_queue *q, + u32 *qid, u32 cid, u16 opcode, + u8 udma_idx) +{ + struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); + struct ionic_admin_ctx admin = { + .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), + .cmd.rdma_queue = { + .opcode = opcode, + .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), + .qid_ver = cpu_to_le32(*qid), + .cid = cpu_to_le32(cid), + .dbid = cpu_to_le16(dev->lif_cfg.dbid), + .depth_log2 = q->depth_log2, + .stride_log2 = q->stride_log2, + .dma_addr = cpu_to_le64(q->dma), + }, + }; + int rc; + + if (ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { + struct ionic_admin_create_cq_resp resp_buf; + + /* Indicates use of CQ create V2 response format. */ + admin.cmd.rdma_queue.qid_ver = cpu_to_le32(IONIC_CREATE_CQ_CMD_V2_MAGIC); + admin.cmd.rdma_queue.udma_idx = udma_idx; + + rc = ionic_rdma_devcmd(dev, &admin); + if (rc) + return rc; + + memcpy(&resp_buf, admin.comp.comp.cmd_data, sizeof(resp_buf)); + *qid = le32_to_cpu(resp_buf.id); + } else { + rc = ionic_rdma_devcmd(dev, &admin); + if (rc) + return rc; + } + + return 0; +} + static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context) { struct ionic_aq *aq = cq_context; @@ -533,12 +576,16 @@ static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev, cq = &vcq->cq[0]; rc = ionic_create_cq_common(vcq, &buf, &attr, NULL, NULL, - NULL, NULL, 0); + NULL, 0); if (rc) goto err_init; - rc = ionic_rdma_queue_devcmd(dev, &cq->q, cq->cqid, cq->eqid, - IONIC_CMD_RDMA_CREATE_CQ); + rc = ionic_rdma_cq_devcmd(vcq, &cq->q, &cq->cqid, cq->eqid, + IONIC_CMD_RDMA_CREATE_CQ, 0); + if (rc) + goto err_cmd; + + rc = ionic_post_create_cq_cmd(cq, NULL, NULL); if (rc) goto err_cmd; @@ -552,6 +599,14 @@ static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev, return ERR_PTR(rc); } +static void ionic_destroy_rdma_admincq(struct ionic_ibdev *dev, + struct ionic_vcq *vcq) +{ + ionic_pre_destroy_cq_cmd(dev, &vcq->cq[0]); + ionic_destroy_cq_common(dev, &vcq->cq[0]); + kfree(vcq); +} + static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev, u32 aqid, u32 cqid) { @@ -890,6 +945,49 @@ static void ionic_qp_event(struct ionic_ibdev *dev, u32 qpid, u8 code) kref_put(&qp->qp_kref, ionic_qp_complete); } +static void ionic_srq_event(struct ionic_ibdev *dev, u32 srqid, u8 code) +{ + unsigned long irqflags; + struct ionic_srq *srq; + struct ib_event ibev; + + xa_lock_irqsave(&dev->srq_tbl, irqflags); + srq = xa_load(&dev->srq_tbl, srqid); + if (srq) + kref_get(&srq->kref); + xa_unlock_irqrestore(&dev->srq_tbl, irqflags); + + if (!srq) { + ibdev_dbg(&dev->ibdev, + "missing srqid %#x code %u\n", srqid, code); + return; + } + + ibev.device = &dev->ibdev; + ibev.element.srq = &srq->ibsrq; + + switch (code) { + case IONIC_V1_EQE_SRQ_LIMIT_REACHED: + ibev.event = IB_EVENT_SRQ_LIMIT_REACHED; + break; + + case IONIC_V1_EQE_SRQ_ERR: + ibev.event = IB_EVENT_SRQ_ERR; + break; + + default: + ibdev_dbg(&dev->ibdev, + "unrecognized srqid %#x code %u\n", srqid, code); + goto out; + } + + if (srq->ibsrq.event_handler) + srq->ibsrq.event_handler(&ibev, srq->ibsrq.srq_context); + +out: + kref_put(&srq->kref, ionic_srq_complete); +} + static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget) { struct ionic_ibdev *dev = eq->dev; @@ -923,6 +1021,10 @@ static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget) ionic_qp_event(dev, qid, code); break; + case IONIC_V1_EQE_TYPE_SRQ: + ionic_srq_event(dev, qid, code); + break; + default: ibdev_dbg(&dev->ibdev, "unknown event %#x type %u\n", evt, type); @@ -1153,8 +1255,7 @@ int ionic_create_rdma_admin(struct ionic_ibdev *dev) vcq->cq[0].cqid); if (IS_ERR(aq)) { /* Clean up the dangling CQ */ - ionic_destroy_cq_common(dev, &vcq->cq[0]); - kfree(vcq); + ionic_destroy_rdma_admincq(dev, vcq); rc = PTR_ERR(aq); @@ -1207,10 +1308,8 @@ void ionic_destroy_rdma_admin(struct ionic_ibdev *dev) cancel_work_sync(&aq->work); __ionic_destroy_rdma_adminq(dev, aq); - if (vcq) { - ionic_destroy_cq_common(dev, &vcq->cq[0]); - kfree(vcq); - } + if (vcq) + ionic_destroy_rdma_admincq(dev, vcq); } kfree(dev->aq_vec); diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c index a70ef59a8064..18ad967812f5 100644 --- a/drivers/infiniband/hw/ionic/ionic_controlpath.c +++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c @@ -79,12 +79,10 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, struct ionic_ctx *ctx, struct ib_udata *udata, struct ionic_qdesc *req_cq, - __u32 *resp_cqid, int udma_idx) { struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); struct ionic_cq *cq = &vcq->cq[udma_idx]; - void *entry; int rc; cq->vcq = vcq; @@ -94,9 +92,11 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, goto err_args; } - rc = ionic_get_cqid(dev, &cq->cqid, udma_idx); - if (rc) - goto err_args; + if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { + rc = ionic_get_cqid(dev, &cq->cqid, udma_idx); + if (rc) + goto err_args; + } cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx); @@ -110,8 +110,9 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, if (rc) goto err_qdesc; - cq->umem = ib_umem_get_va(&dev->ibdev, req_cq->addr, - req_cq->size, IB_ACCESS_LOCAL_WRITE); + cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, + req_cq->addr, req_cq->size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { rc = PTR_ERR(cq->umem); goto err_qdesc; @@ -122,8 +123,6 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, cq->q.mask = req_cq->mask; cq->q.depth_log2 = req_cq->depth_log2; cq->q.stride_log2 = req_cq->stride_log2; - - *resp_cqid = cq->cqid; } else { rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev, attr->cqe + IONIC_CQ_GRACE, @@ -131,7 +130,6 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, if (rc) goto err_q_init; - ionic_queue_dbell_init(&cq->q, cq->cqid); cq->color = true; cq->credit = cq->q.mask; } @@ -143,20 +141,8 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, init_completion(&cq->cq_rel_comp); kref_init(&cq->cq_kref); - entry = xa_store_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL); - if (entry) { - if (!xa_is_err(entry)) - rc = -EINVAL; - else - rc = xa_err(entry); - - goto err_xa; - } - return 0; -err_xa: - ionic_pgtbl_unbuf(dev, buf); err_pgtbl_init: if (!udata) ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); @@ -164,29 +150,52 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, if (cq->umem) ib_umem_release(cq->umem); err_qdesc: - ionic_put_cqid(dev, cq->cqid); + if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) + ionic_put_cqid(dev, cq->cqid); err_args: cq->vcq = NULL; return rc; } +int ionic_post_create_cq_cmd(struct ionic_cq *cq, + struct ib_udata *udata, + __u32 *resp_cqid) +{ + struct ionic_ibdev *dev = to_ionic_ibdev(cq->vcq->ibcq.device); + + if (udata && resp_cqid) + *resp_cqid = cq->cqid; + else + ionic_queue_dbell_init(&cq->q, cq->cqid); + + return xa_insert_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL); +} + +void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq) +{ + if (!cq->vcq) + return; + + if (!xa_erase_irq(&dev->cq_tbl, cq->cqid)) + return; + + kref_put(&cq->cq_kref, ionic_cq_complete); + wait_for_completion(&cq->cq_rel_comp); +} + void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq) { if (!cq->vcq) return; - xa_erase_irq(&dev->cq_tbl, cq->cqid); - - kref_put(&cq->cq_kref, ionic_cq_complete); - wait_for_completion(&cq->cq_rel_comp); - if (cq->umem) ib_umem_release(cq->umem); else ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev); - ionic_put_cqid(dev, cq->cqid); + if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) + ionic_put_cqid(dev, cq->cqid); cq->vcq = NULL; } @@ -418,6 +427,8 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) resp.udma_count = dev->lif_cfg.udma_count; resp.expdb_mask = dev->lif_cfg.expdb_mask; + resp.rcq_sign_bit = dev->lif_cfg.rcq_sign_bit; + resp.comp_mask |= IONIC_CTX_CMASK_IONIC_FLAGS; if (dev->lif_cfg.sq_expdb) resp.expdb_qtypes |= IONIC_EXPDB_SQ; @@ -1220,8 +1231,11 @@ int ionic_dealloc_mw(struct ib_mw *ibmw) static int ionic_create_cq_cmd(struct ionic_ibdev *dev, struct ionic_ctx *ctx, struct ionic_cq *cq, - struct ionic_tbl_buf *buf) + struct ionic_tbl_buf *buf, + int udma_idx, + int *out_udma_idx) { + struct ionic_admin_create_cq_resp *resp_buf; const u16 dbid = ionic_ctx_dbid(dev, ctx); struct ionic_admin_wr wr = { .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), @@ -1241,13 +1255,57 @@ static int ionic_create_cq_cmd(struct ionic_ibdev *dev, } } }; + dma_addr_t resp_buf_dma; + int rc; if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ) return -EBADRQC; - ionic_admin_post(dev, &wr); + if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { + ionic_admin_post(dev, &wr); + return ionic_admin_wait(dev, &wr, 0); + } - return ionic_admin_wait(dev, &wr, 0); + resp_buf = kzalloc_obj(*resp_buf); + if (!resp_buf) + return -ENOMEM; + + resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, + sizeof(*resp_buf), + DMA_FROM_DEVICE); + + rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); + if (rc) { + rc = -ENOMEM; + goto err_dma; + } + + wr.wqe.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V2_LEN); + wr.wqe.cmd.create_cq.udma_idx = udma_idx; + wr.wqe.cmd.create_cq.resp_dma_addr = cpu_to_le64(resp_buf_dma); + wr.wqe.cmd.create_cq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN); + + ionic_admin_post(dev, &wr); + rc = ionic_admin_wait(dev, &wr, 0); + + dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), + DMA_FROM_DEVICE); + if (rc) + goto err_dma; + + if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN) { + rc = -EOPNOTSUPP; + goto err_dma; + } + + cq->cqid = le32_to_cpu(resp_buf->id); + if (out_udma_idx) + *out_udma_idx = resp_buf->udma_idx; + +err_dma: + kfree(resp_buf); + + return rc; } static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid) @@ -1306,16 +1364,21 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata, &req.cq[udma_idx], - &resp.cqid[udma_idx], udma_idx); if (rc) goto err_init; - rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf); + rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf, + udma_idx, NULL); if (rc) goto err_cmd; ionic_pgtbl_unbuf(dev, &buf); + + rc = ionic_post_create_cq_cmd(&vcq->cq[udma_idx], udata, + &resp.cqid[udma_idx]); + if (rc) + goto err_post; } vcq->ibcq.cqe = attr->cqe; @@ -1335,6 +1398,8 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, --udma_idx; if (!(vcq->udma_mask & BIT(udma_idx))) continue; + ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]); +err_post: ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); err_cmd: ionic_pgtbl_unbuf(dev, &buf); @@ -1362,6 +1427,13 @@ int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) if (!(vcq->udma_mask & BIT(udma_idx))) continue; + /* + * Untrack the CQ before releasing its hardware ID below, so a + * concurrent create that gets the same ID reused by firmware + * cannot have its fresh XArray entry erased by this destroy. + */ + ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]); + rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid); if (rc_tmp) { if (!rc) @@ -1388,13 +1460,14 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, struct ionic_qp *qp, struct ionic_tbl_buf *sq_buf, struct ionic_tbl_buf *rq_buf, - struct ib_qp_init_attr *attr) + struct ib_qp_init_attr *attr, + u32 ionic_flags) { const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject); const u32 flags = to_ionic_qp_flags(0, 0, qp->sq_cmb & IONIC_CMB_ENABLE, - qp->rq_cmb & IONIC_CMB_ENABLE, - qp->sq_spec, qp->rq_spec, + qp->rq.cmb & IONIC_CMB_ENABLE, + qp->sq_spec, qp->rq.spec, pd->flags & IONIC_QPF_PRIVILEGED, pd_remote_privileged(&pd->ibpd)); struct ionic_admin_wr wr = { @@ -1404,7 +1477,12 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, .len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN), .cmd.create_qp = { .pd_id = cpu_to_le32(pd->pdid), - .priv_flags = cpu_to_be32(flags), + /* User-supplied ionic_flags are passed through to + * firmware, which validates and rejects any + * unsupported or unauthorized bits. + */ + .priv_flags = cpu_to_be32(flags | + (ionic_flags & IONIC_QP_USER_FLAGS_MASK)), .type_state = to_ionic_qp_type(attr->qp_type), .dbid_flags = cpu_to_le16(dbid), .id_ver = cpu_to_le32(qp->qpid), @@ -1426,15 +1504,18 @@ static int ionic_create_qp_cmd(struct ionic_ibdev *dev, wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0); } - if (qp->has_rq) { + if (!qp->srq) { wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); - wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.depth_log2; - wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.stride_log2; + wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.q.depth_log2; + wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.q.stride_log2; wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2; wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0); wr.wqe.cmd.create_qp.rq_map_count = cpu_to_le32(rq_buf->tbl_pages); wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0); + } else { + wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(qp->srq->rq.qid); + wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid); } ionic_admin_post(dev, &wr); @@ -1451,8 +1532,8 @@ static int ionic_modify_qp_cmd(struct ionic_ibdev *dev, const u32 flags = to_ionic_qp_flags(attr->qp_access_flags, attr->en_sqd_async_notify, qp->sq_cmb & IONIC_CMB_ENABLE, - qp->rq_cmb & IONIC_CMB_ENABLE, - qp->sq_spec, qp->rq_spec, + qp->rq.cmb & IONIC_CMB_ENABLE, + qp->sq_spec, qp->rq.spec, pd->flags & IONIC_QPF_PRIVILEGED, pd_remote_privileged(qp->ibqp.pd)); const u8 state = to_ionic_qp_modify_state(attr->qp_state, @@ -1620,11 +1701,11 @@ static int ionic_query_qp_cmd(struct ionic_ibdev *dev, ionic_v1_send_wqe_max_data(qp->sq.stride_log2, expdb); } - if (qp->has_rq) { + if (!qp->srq) { attr->cap.max_recv_sge = - ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, - qp->rq_spec, - qp->rq_cmb & IONIC_CMB_EXPDB); + ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, + qp->rq.spec, + qp->rq.cmb & IONIC_CMB_EXPDB); } query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); @@ -1994,99 +2075,123 @@ static void ionic_qp_sq_destroy(struct ionic_ibdev *dev, ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev); } -static void ionic_qp_rq_init_cmb(struct ionic_ibdev *dev, - struct ionic_qp *qp, - struct ib_udata *udata) +static void ionic_rq_mmap_cmb(struct ionic_ibdev *dev, + struct ionic_ctx *ctx, + struct ionic_rq *rq, + u64 *cmb_offset) +{ + bool wc; + + /* set mapping by default to uncached for + * expdb (to guarantee writes order) otherwise + * writecombine, unless this default is + * overridden by userspace + */ + if ((rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == + (IONIC_CMB_WC | IONIC_CMB_UC)) { + ibdev_dbg(&dev->ibdev, + "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC set, using default driver mapping\n"); + rq->cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); + } + + if (rq->cmb & IONIC_CMB_EXPDB) + wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == IONIC_CMB_WC; + else + wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) != IONIC_CMB_UC; + + /* let userspace know the mapping */ + if (wc) + rq->cmb |= IONIC_CMB_WC; + else + rq->cmb |= IONIC_CMB_UC; + + rq->mmap_cmb = ionic_mmap_entry_insert(ctx, rq->q.size, + PHYS_PFN(rq->cmb_addr), + wc ? IONIC_MMAP_WC : 0, + cmb_offset); +} + +static void ionic_rq_init_cmb(struct ionic_ibdev *dev, + struct ionic_rq *rq, + struct ib_udata *udata) { u8 expdb_stride_log2 = 0; bool expdb; int rc; - if (!(qp->rq_cmb & IONIC_CMB_ENABLE)) + if (!(rq->cmb & IONIC_CMB_ENABLE)) goto not_in_cmb; - if (qp->rq_cmb & ~IONIC_CMB_SUPPORTED) { - if (qp->rq_cmb & IONIC_CMB_REQUIRE) + if (rq->cmb & ~IONIC_CMB_SUPPORTED) { + if (rq->cmb & IONIC_CMB_REQUIRE) goto not_in_cmb; - qp->rq_cmb &= IONIC_CMB_SUPPORTED; + rq->cmb &= IONIC_CMB_SUPPORTED; } - if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) { - if (qp->rq_cmb & IONIC_CMB_REQUIRE) + if ((rq->cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) { + if (rq->cmb & IONIC_CMB_REQUIRE) goto not_in_cmb; - qp->rq_cmb &= ~IONIC_CMB_EXPDB; + rq->cmb &= ~IONIC_CMB_EXPDB; } - qp->rq_cmb_order = order_base_2(qp->rq.size / PAGE_SIZE); + rq->cmb_order = order_base_2(rq->q.size / PAGE_SIZE); - if (qp->rq_cmb_order >= IONIC_RQCMB_ORDER) + if (rq->cmb_order >= IONIC_RQCMB_ORDER) goto not_in_cmb; - if (qp->rq_cmb & IONIC_CMB_EXPDB) - expdb_stride_log2 = qp->rq.stride_log2; + if (rq->cmb & IONIC_CMB_EXPDB) + expdb_stride_log2 = rq->q.stride_log2; - rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->rq_cmb_pgid, - &qp->rq_cmb_addr, qp->rq_cmb_order, + rc = ionic_get_cmb(dev->lif_cfg.lif, &rq->cmb_pgid, + &rq->cmb_addr, rq->cmb_order, expdb_stride_log2, &expdb); if (rc) goto not_in_cmb; - if ((qp->rq_cmb & IONIC_CMB_EXPDB) && !expdb) { - if (qp->rq_cmb & IONIC_CMB_REQUIRE) + if ((rq->cmb & IONIC_CMB_EXPDB) && !expdb) { + if (rq->cmb & IONIC_CMB_REQUIRE) goto err_map; - qp->rq_cmb &= ~IONIC_CMB_EXPDB; + rq->cmb &= ~IONIC_CMB_EXPDB; } return; err_map: - ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order); + ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order); not_in_cmb: - if (qp->rq_cmb & IONIC_CMB_REQUIRE) + if (rq->cmb & IONIC_CMB_REQUIRE) ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n"); - qp->rq_cmb = 0; - qp->rq_cmb_order = IONIC_RES_INVALID; - qp->rq_cmb_pgid = 0; - qp->rq_cmb_addr = 0; + rq->cmb = 0; + rq->cmb_order = IONIC_RES_INVALID; + rq->cmb_pgid = 0; + rq->cmb_addr = 0; } -static void ionic_qp_rq_destroy_cmb(struct ionic_ibdev *dev, - struct ionic_ctx *ctx, - struct ionic_qp *qp) +static void ionic_rq_destroy_cmb(struct ionic_ibdev *dev, + struct ionic_ctx *ctx, + struct ionic_rq *rq) { - if (!(qp->rq_cmb & IONIC_CMB_ENABLE)) + if (!(rq->cmb & IONIC_CMB_ENABLE)) return; if (ctx) - rdma_user_mmap_entry_remove(qp->mmap_rq_cmb); + rdma_user_mmap_entry_remove(rq->mmap_cmb); - ionic_put_cmb(dev->lif_cfg.lif, qp->rq_cmb_pgid, qp->rq_cmb_order); + ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order); } -static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, - struct ionic_qp *qp, struct ionic_qdesc *rq, - struct ionic_tbl_buf *buf, int max_wr, int max_sge, - int rq_spec, struct ib_udata *udata) +static int ionic_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, + struct ionic_rq *rq, struct ionic_qdesc *qdesc, + struct ionic_tbl_buf *buf, int max_wr, int max_sge, + int rq_spec, struct ib_udata *udata) { int rc = 0, i; u32 wqe_size; - if (!qp->has_rq) { - if (buf) { - buf->tbl_buf = NULL; - buf->tbl_limit = 0; - buf->tbl_pages = 0; - } - if (udata) - rc = ionic_validate_qdesc_zero(rq); - - return rc; - } - rc = -EINVAL; if (max_wr < 0 || max_wr > 0xffff) @@ -2100,105 +2205,102 @@ static int ionic_qp_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx, return rc; if (udata) { - rc = ionic_validate_qdesc(rq); + rc = ionic_validate_qdesc(qdesc); if (rc) return rc; - qp->rq_spec = rq_spec; + rq->spec = rq_spec; - qp->rq.ptr = NULL; - qp->rq.size = rq->size; - qp->rq.mask = rq->mask; - qp->rq.depth_log2 = rq->depth_log2; - qp->rq.stride_log2 = rq->stride_log2; + rq->q.ptr = NULL; + rq->q.size = qdesc->size; + rq->q.mask = qdesc->mask; + rq->q.depth_log2 = qdesc->depth_log2; + rq->q.stride_log2 = qdesc->stride_log2; - qp->rq_meta = NULL; + rq->meta = NULL; - qp->rq_umem = ib_umem_get_va(&dev->ibdev, rq->addr, rq->size, 0); - if (IS_ERR(qp->rq_umem)) - return PTR_ERR(qp->rq_umem); + rq->umem = ib_umem_get_va(&dev->ibdev, qdesc->addr, qdesc->size, 0); + if (IS_ERR(rq->umem)) + return PTR_ERR(rq->umem); } else { - qp->rq_umem = NULL; + rq->umem = NULL; - qp->rq_spec = ionic_v1_use_spec_sge(max_sge, rq_spec); - if (rq_spec && !qp->rq_spec) + rq->spec = ionic_v1_use_spec_sge(max_sge, rq_spec); + if (rq_spec && !rq->spec) ibdev_dbg(&dev->ibdev, "init rq: max_sge %u disables spec\n", max_sge); - if (qp->rq_cmb & IONIC_CMB_EXPDB) { + if (rq->cmb & IONIC_CMB_EXPDB) { wqe_size = ionic_v1_recv_wqe_min_size(max_sge, - qp->rq_spec, + rq->spec, true); if (!ionic_expdb_wqe_size_supported(dev, wqe_size)) - qp->rq_cmb &= ~IONIC_CMB_EXPDB; + rq->cmb &= ~IONIC_CMB_EXPDB; } - if (!(qp->rq_cmb & IONIC_CMB_EXPDB)) + if (!(rq->cmb & IONIC_CMB_EXPDB)) wqe_size = ionic_v1_recv_wqe_min_size(max_sge, - qp->rq_spec, + rq->spec, false); - rc = ionic_queue_init(&qp->rq, dev->lif_cfg.hwdev, + rc = ionic_queue_init(&rq->q, dev->lif_cfg.hwdev, max_wr, wqe_size); if (rc) return rc; - ionic_queue_dbell_init(&qp->rq, qp->qpid); + ionic_queue_dbell_init(&rq->q, rq->qid); - qp->rq_meta = kmalloc_objs(*qp->rq_meta, (u32)qp->rq.mask + 1); - if (!qp->rq_meta) { + rq->meta = kmalloc_objs(*rq->meta, (u32)rq->q.mask + 1); + if (!rq->meta) { rc = -ENOMEM; goto err_rq_meta; } - for (i = 0; i < qp->rq.mask; ++i) - qp->rq_meta[i].next = &qp->rq_meta[i + 1]; - qp->rq_meta[i].next = IONIC_META_LAST; - qp->rq_meta_head = &qp->rq_meta[0]; + for (i = 0; i < rq->q.mask; ++i) + rq->meta[i].next = &rq->meta[i + 1]; + rq->meta[i].next = IONIC_META_LAST; + rq->meta_head = &rq->meta[0]; } - ionic_qp_rq_init_cmb(dev, qp, udata); + ionic_rq_init_cmb(dev, rq, udata); - if (qp->rq_cmb & IONIC_CMB_ENABLE) + if (rq->cmb & IONIC_CMB_ENABLE) rc = ionic_pgtbl_init(dev, buf, NULL, - (u64)qp->rq_cmb_pgid << PAGE_SHIFT, + (u64)rq->cmb_pgid << PAGE_SHIFT, 1, PAGE_SIZE); else - rc = ionic_pgtbl_init(dev, buf, - qp->rq_umem, qp->rq.dma, 1, PAGE_SIZE); + rc = ionic_pgtbl_init(dev, buf, rq->umem, + rq->q.dma, 1, PAGE_SIZE); if (rc) goto err_rq_tbl; return 0; err_rq_tbl: - ionic_qp_rq_destroy_cmb(dev, ctx, qp); - kfree(qp->rq_meta); + ionic_rq_destroy_cmb(dev, ctx, rq); + kfree(rq->meta); err_rq_meta: - if (qp->rq_umem) - ib_umem_release(qp->rq_umem); + if (rq->umem) + ib_umem_release(rq->umem); else - ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev); + ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); return rc; } -static void ionic_qp_rq_destroy(struct ionic_ibdev *dev, - struct ionic_ctx *ctx, - struct ionic_qp *qp) +static void ionic_rq_destroy(struct ionic_ibdev *dev, + struct ionic_ctx *ctx, + struct ionic_rq *rq) { - if (!qp->has_rq) - return; + ionic_rq_destroy_cmb(dev, ctx, rq); - ionic_qp_rq_destroy_cmb(dev, ctx, qp); + kfree(rq->meta); - kfree(qp->rq_meta); - - if (qp->rq_umem) - ib_umem_release(qp->rq_umem); + if (rq->umem) + ib_umem_release(rq->umem); else - ionic_queue_destroy(&qp->rq, dev->lif_cfg.hwdev); + ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev); } int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, @@ -2221,6 +2323,12 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, rc = ib_copy_validate_udata_in(udata, req, rsvd); if (rc) return rc; + + if (req.ionic_flags & ~IONIC_QP_USER_FLAGS_MASK) + return -EINVAL; + + if (req.rsvd_pad) + return -EINVAL; } else { req.sq_spec = IONIC_SPEC_HIGH; req.rq_spec = IONIC_SPEC_HIGH; @@ -2233,25 +2341,32 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, INIT_LIST_HEAD(&qp->cq_poll_sq); INIT_LIST_HEAD(&qp->cq_flush_sq); - INIT_LIST_HEAD(&qp->cq_flush_rq); + INIT_LIST_HEAD(&qp->rq.cq_flush); spin_lock_init(&qp->sq_lock); - spin_lock_init(&qp->rq_lock); + spin_lock_init(&qp->rq.lock); qp->has_sq = 1; - qp->has_rq = 1; + + if (attr->srq) + qp->srq = to_ionic_srq(attr->srq); + else + qp->srq = NULL; if (attr->qp_type == IB_QPT_GSI) { rc = ionic_get_gsi_qpid(dev, &qp->qpid); } else { udma_mask = BIT(dev->lif_cfg.udma_count) - 1; - if (qp->has_sq) + if (attr->send_cq) udma_mask &= to_ionic_vcq(attr->send_cq)->udma_mask; - if (qp->has_rq) + if (attr->recv_cq) udma_mask &= to_ionic_vcq(attr->recv_cq)->udma_mask; + if (attr->srq) + udma_mask &= BIT(to_ionic_srq(attr->srq)->udma_idx); + if (udata && req.udma_mask) udma_mask &= req.udma_mask; @@ -2278,13 +2393,8 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, goto err_ahid; } - if (udata) { - if (req.rq_cmb & IONIC_CMB_ENABLE) - qp->rq_cmb = req.rq_cmb; - - if (req.sq_cmb & IONIC_CMB_ENABLE) - qp->sq_cmb = req.sq_cmb; - } + if (udata && (req.sq_cmb & IONIC_CMB_ENABLE)) + qp->sq_cmb = req.sq_cmb; rc = ionic_qp_sq_init(dev, ctx, qp, &req.sq, &sq_buf, attr->cap.max_send_wr, attr->cap.max_send_sge, @@ -2292,16 +2402,30 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, if (rc) goto err_sq; - rc = ionic_qp_rq_init(dev, ctx, qp, &req.rq, &rq_buf, - attr->cap.max_recv_wr, attr->cap.max_recv_sge, - req.rq_spec, udata); + if (qp->srq) { + rq_buf.tbl_buf = NULL; + rq_buf.tbl_limit = 0; + rq_buf.tbl_pages = 0; + + if (udata) + rc = ionic_validate_qdesc_zero(&req.rq); + } else { + if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) + qp->rq.cmb = req.rq_cmb; + + /* for non-srq qps, rq qid is same as qpid */ + qp->rq.qid = qp->qpid; + rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf, + attr->cap.max_recv_wr, attr->cap.max_recv_sge, + req.rq_spec, udata); + } if (rc) goto err_rq; rc = ionic_create_qp_cmd(dev, pd, to_ionic_vcq_cq(attr->send_cq, qp->udma_idx), to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx), - qp, &sq_buf, &rq_buf, attr); + qp, &sq_buf, &rq_buf, attr, req.ionic_flags); if (rc) goto err_cmd; @@ -2336,52 +2460,26 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, &resp.sq_cmb_offset); if (!qp->mmap_sq_cmb) { rc = -ENOMEM; - goto err_mmap_sq; + goto err_qp_cmd; } resp.sq_cmb = qp->sq_cmb; } - if (qp->rq_cmb & IONIC_CMB_ENABLE) { - bool wc; + if (!qp->srq && (qp->rq.cmb & IONIC_CMB_ENABLE)) { + ionic_rq_mmap_cmb(dev, ctx, &qp->rq, &resp.rq_cmb_offset); - if ((qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == - (IONIC_CMB_WC | IONIC_CMB_UC)) { - ibdev_dbg(&dev->ibdev, - "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n"); - qp->rq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC); - } - - if (qp->rq_cmb & IONIC_CMB_EXPDB) - wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) - == IONIC_CMB_WC; - else - wc = (qp->rq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) - != IONIC_CMB_UC; - - /* let userspace know the mapping */ - if (wc) - qp->rq_cmb |= IONIC_CMB_WC; - else - qp->rq_cmb |= IONIC_CMB_UC; - - qp->mmap_rq_cmb = - ionic_mmap_entry_insert(ctx, - qp->rq.size, - PHYS_PFN(qp->rq_cmb_addr), - wc ? IONIC_MMAP_WC : 0, - &resp.rq_cmb_offset); - if (!qp->mmap_rq_cmb) { + if (!qp->rq.mmap_cmb) { rc = -ENOMEM; - goto err_mmap_rq; + goto err_qp_cmd; } - resp.rq_cmb = qp->rq_cmb; + resp.rq_cmb = qp->rq.cmb; } rc = ib_respond_udata(udata, resp); if (rc) - goto err_resp; + goto err_qp_cmd; } ionic_pgtbl_unbuf(dev, &rq_buf); @@ -2399,7 +2497,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, else rc = xa_err(entry); - goto err_resp; + goto err_qp_cmd; } if (qp->has_sq) { @@ -2417,30 +2515,26 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, qp->sq_cqid = cq->cqid; } - if (qp->has_rq) { + if (!qp->srq) { cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx); - attr->cap.max_recv_wr = qp->rq.mask; + attr->cap.max_recv_wr = qp->rq.q.mask; attr->cap.max_recv_sge = - ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, - qp->rq_spec, - qp->rq_cmb & IONIC_CMB_EXPDB); - qp->rq_cqid = cq->cqid; + ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, + qp->rq.spec, + qp->rq.cmb & IONIC_CMB_EXPDB); + qp->rq.cqid = cq->cqid; } return 0; -err_resp: - if (udata && (qp->rq_cmb & IONIC_CMB_ENABLE)) - rdma_user_mmap_entry_remove(qp->mmap_rq_cmb); -err_mmap_rq: - if (udata && (qp->sq_cmb & IONIC_CMB_ENABLE)) - rdma_user_mmap_entry_remove(qp->mmap_sq_cmb); -err_mmap_sq: +err_qp_cmd: ionic_destroy_qp_cmd(dev, qp->qpid); err_cmd: - ionic_pgtbl_unbuf(dev, &rq_buf); - ionic_qp_rq_destroy(dev, ctx, qp); + if (!qp->srq) { + ionic_pgtbl_unbuf(dev, &rq_buf); + ionic_rq_destroy(dev, ctx, &qp->rq); + } err_rq: ionic_pgtbl_unbuf(dev, &sq_buf); ionic_qp_sq_destroy(dev, ctx, qp); @@ -2491,18 +2585,18 @@ void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) spin_unlock_irqrestore(&cq->lock, irqflags); } - if (qp->ibqp.recv_cq) { + if (qp->ibqp.recv_cq && !qp->srq) { cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); - /* Hold the CQ lock and QP rq_lock to set up flush */ + /* Hold the CQ lock and QP rq.lock to set up flush */ spin_lock_irqsave(&cq->lock, irqflags); - spin_lock(&qp->rq_lock); - qp->rq_flush = true; - if (!ionic_queue_empty(&qp->rq)) { + spin_lock(&qp->rq.lock); + qp->rq.flush = true; + if (!ionic_queue_empty(&qp->rq.q)) { cq->flush = true; - list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); + list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); } - spin_unlock(&qp->rq_lock); + spin_unlock(&qp->rq.lock); spin_unlock_irqrestore(&cq->lock, irqflags); } } @@ -2567,18 +2661,18 @@ static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp) spin_unlock(&qp->sq_lock); } - if (qp->has_rq) { - spin_lock(&qp->rq_lock); - qp->rq_flush = false; - qp->rq.prod = 0; - qp->rq.cons = 0; - if (qp->rq_meta) { - for (i = 0; i < qp->rq.mask; ++i) - qp->rq_meta[i].next = &qp->rq_meta[i + 1]; - qp->rq_meta[i].next = IONIC_META_LAST; + if (!qp->srq) { + spin_lock(&qp->rq.lock); + qp->rq.flush = false; + qp->rq.q.prod = 0; + qp->rq.q.cons = 0; + if (qp->rq.meta) { + for (i = 0; i < qp->rq.q.mask; ++i) + qp->rq.meta[i].next = &qp->rq.meta[i + 1]; + qp->rq.meta[i].next = IONIC_META_LAST; } - qp->rq_meta_head = &qp->rq_meta[0]; - spin_unlock(&qp->rq_lock); + qp->rq.meta_head = &qp->rq.meta[0]; + spin_unlock(&qp->rq.lock); } local_irq_restore(irqflags); @@ -2695,8 +2789,8 @@ int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (qp->has_sq) attr->cap.max_send_wr = qp->sq.mask; - if (qp->has_rq) - attr->cap.max_recv_wr = qp->rq.mask; + if (!qp->srq) + attr->cap.max_recv_wr = qp->rq.q.mask; init_attr->event_handler = ibqp->event_handler; init_attr->qp_context = ibqp->qp_context; @@ -2752,11 +2846,13 @@ int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx); spin_lock_irqsave(&cq->lock, irqflags); ionic_clean_cq(cq, qp->qpid); - list_del(&qp->cq_flush_rq); + list_del(&qp->rq.cq_flush); spin_unlock_irqrestore(&cq->lock, irqflags); } - ionic_qp_rq_destroy(dev, ctx, qp); + if (!qp->srq) + ionic_rq_destroy(dev, ctx, &qp->rq); + ionic_qp_sq_destroy(dev, ctx, qp); if (qp->has_ah) { ionic_put_ahid(dev, qp->ahid); @@ -2766,3 +2862,279 @@ int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) return 0; } + +static int ionic_create_srq_cmd(struct ionic_ibdev *dev, + struct ionic_ctx *ctx, + struct ionic_srq *srq, + struct ionic_pd *pd, + struct ionic_tbl_buf *buf, + u8 udma_mask) +{ + const u16 flags = (srq->rq.cmb & IONIC_CMB_ENABLE) ? IONIC_SRQF_CMB : 0; + struct ionic_admin_create_srq_resp *resp_buf; + const u16 dbid = ionic_ctx_dbid(dev, ctx); + struct ionic_admin_wr wr = { + .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), + .wqe = { + .op = IONIC_V1_ADMIN_CREATE_SRQ, + .len = cpu_to_le16(IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN), + .cmd.create_srq = { + .pd_id = cpu_to_le32(pd->pdid), + .depth_log2 = srq->rq.q.depth_log2, + .stride_log2 = srq->rq.q.stride_log2, + .page_size_log2 = buf->page_size_log2, + .map_count = cpu_to_le32(buf->tbl_pages), + .dma_addr = ionic_pgtbl_dma(buf, 0), + .dbid = cpu_to_le16(dbid), + .qid = cpu_to_le32(srq->rq.qid), + .low_wqes_limit = cpu_to_le16(srq->srq_limit), + .udma_mask = udma_mask, + .flags = cpu_to_le16(flags), + } + } + }; + dma_addr_t resp_buf_dma; + int rc; + + if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_SRQ) + return -EOPNOTSUPP; + + resp_buf = kzalloc_obj(*resp_buf); + if (!resp_buf) + return -ENOMEM; + + resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf, + sizeof(*resp_buf), + DMA_FROM_DEVICE); + + rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma); + if (rc) + goto err_dma; + + wr.wqe.cmd.create_srq.resp_dma_addr = cpu_to_le64(resp_buf_dma); + wr.wqe.cmd.create_srq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN); + + ionic_admin_post(dev, &wr); + + rc = ionic_admin_wait(dev, &wr, 0); + + dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf), + DMA_FROM_DEVICE); + if (rc) + goto err_dma; + + if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN) { + rc = -EOPNOTSUPP; + goto err_dma; + } + + srq->rq.qid = le32_to_cpu(resp_buf->id); + srq->udma_idx = resp_buf->udma_idx; + +err_dma: + kfree(resp_buf); + + return rc; +} + +static int ionic_destroy_srq_cmd(struct ionic_ibdev *dev, u32 srqid) +{ + struct ionic_admin_wr wr = { + .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), + .wqe = { + .op = IONIC_V1_ADMIN_DESTROY_SRQ, + .len = cpu_to_le16(IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN), + .cmd.destroy_srq = { + .qid = cpu_to_le32(srqid), + }, + } + }; + + if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_SRQ) + return -EOPNOTSUPP; + + ionic_admin_post(dev, &wr); + + return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN); +} + +static int ionic_modify_srq_cmd(struct ionic_ibdev *dev, u32 srqid, + u16 srq_limit) +{ + struct ionic_admin_wr wr = { + .work = COMPLETION_INITIALIZER_ONSTACK(wr.work), + .wqe = { + .op = IONIC_V1_ADMIN_MODIFY_SRQ, + .len = cpu_to_le16(IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN), + .cmd.modify_srq = { + .qid = cpu_to_le32(srqid), + .low_wqes_limit = cpu_to_le16(srq_limit), + }, + } + }; + + if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_SRQ) + return -EOPNOTSUPP; + + ionic_admin_post(dev, &wr); + + return ionic_admin_wait(dev, &wr, 0); +} + +int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, + struct ib_udata *udata) +{ + struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); + struct ionic_pd *pd = to_ionic_pd(ibsrq->pd); + struct ionic_srq *srq = to_ionic_srq(ibsrq); + struct ionic_ctx *ctx = + rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); + struct ionic_srq_resp resp = {}; + struct ionic_srq_req req = {}; + struct ionic_tbl_buf buf = {}; + u8 udma_mask; + int rc; + + if (!ctx) + return -EOPNOTSUPP; + + if (attr->srq_type != IB_SRQT_BASIC) + return -EOPNOTSUPP; + + if (attr->attr.max_sge > IONIC_MAX_SRQ_SGES) + return -EINVAL; + + if (attr->attr.srq_limit >= IONIC_MAX_SRQ_LIMIT) + return -EINVAL; + + udma_mask = BIT(dev->lif_cfg.udma_count) - 1; + if (udata) { + rc = ib_copy_validate_udata_in(udata, req, rsvd); + if (rc) + return rc; + + if (memchr_inv(req.rsvd, 0, sizeof(req.rsvd))) + return -EINVAL; + + udma_mask &= req.udma_mask; + } + + if (!udma_mask) + return -EINVAL; + + if (udata && (req.rq_cmb & IONIC_CMB_ENABLE)) + srq->rq.cmb = req.rq_cmb; + + rc = ionic_rq_init(dev, ctx, &srq->rq, &req.rq, &buf, + attr->attr.max_wr, attr->attr.max_sge, + req.rq_spec, udata); + if (rc) + return rc; + + srq->srq_limit = attr->attr.srq_limit; + + rc = ionic_create_srq_cmd(dev, ctx, srq, pd, &buf, udma_mask); + if (rc) + goto err_cmd; + + ionic_queue_dbell_init(&srq->rq.q, srq->rq.qid); + + if (udata) { + resp.srqid = srq->rq.qid; + resp.udma_idx = srq->udma_idx; + + if (srq->rq.cmb & IONIC_CMB_ENABLE) { + ionic_rq_mmap_cmb(dev, ctx, &srq->rq, &resp.rq_cmb_offset); + if (!srq->rq.mmap_cmb) { + rc = -ENOMEM; + goto err_srq_cmd; + } + + resp.rq_cmb = srq->rq.cmb; + } + + rc = ib_respond_udata(udata, resp); + if (rc) + goto err_srq_cmd; + } + + kref_init(&srq->kref); + init_completion(&srq->rel_comp); + + rc = xa_insert_irq(&dev->srq_tbl, srq->rq.qid, srq, GFP_KERNEL); + if (rc) + goto err_srq_cmd; + + ionic_pgtbl_unbuf(dev, &buf); + + attr->attr.max_wr = srq->rq.q.mask; + + return 0; + +err_srq_cmd: + ionic_destroy_srq_cmd(dev, srq->rq.qid); +err_cmd: + ionic_pgtbl_unbuf(dev, &buf); + ionic_rq_destroy(dev, ctx, &srq->rq); + + return rc; +} + +int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) +{ + struct ionic_ctx *ctx = + rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx); + struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); + struct ionic_srq *srq = to_ionic_srq(ibsrq); + int rc; + + if (xa_erase_irq(&dev->srq_tbl, srq->rq.qid)) { + kref_put(&srq->kref, ionic_srq_complete); + wait_for_completion(&srq->rel_comp); + } + + rc = ionic_destroy_srq_cmd(dev, srq->rq.qid); + if (rc) + return rc; + + ionic_rq_destroy(dev, ctx, &srq->rq); + + return 0; +} + +int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + struct ionic_srq *srq = to_ionic_srq(ibsrq); + + srq_attr->max_wr = srq->rq.q.mask; + srq_attr->max_sge = + ionic_v1_recv_wqe_max_sge(srq->rq.q.stride_log2, srq->rq.spec, + srq->rq.cmb & IONIC_CMB_EXPDB); + srq_attr->srq_limit = srq->srq_limit; + + return 0; +} + +int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device); + struct ionic_srq *srq = to_ionic_srq(ibsrq); + int rc; + + if (attr_mask & IB_SRQ_MAX_WR) + return -EINVAL; + + if (attr_mask & IB_SRQ_LIMIT) { + if (attr->srq_limit >= IONIC_MAX_SRQ_LIMIT) + return -EINVAL; + + rc = ionic_modify_srq_cmd(dev, srq->rq.qid, attr->srq_limit); + if (rc) + return rc; + + srq->srq_limit = attr->srq_limit; + } + + return 0; +} diff --git a/drivers/infiniband/hw/ionic/ionic_datapath.c b/drivers/infiniband/hw/ionic/ionic_datapath.c index 3e2300f7ea10..c5726c6bd5cb 100644 --- a/drivers/infiniband/hw/ionic/ionic_datapath.c +++ b/drivers/infiniband/hw/ionic/ionic_datapath.c @@ -34,17 +34,17 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) struct ionic_v1_wqe *wqe; u64 wqe_idx; - if (!qp->rq_flush) + if (!qp->rq.flush) return 0; - if (ionic_queue_empty(&qp->rq)) + if (ionic_queue_empty(&qp->rq.q)) return 0; - wqe = ionic_queue_at_cons(&qp->rq); + wqe = ionic_queue_at_cons(&qp->rq.q); wqe_idx = le64_to_cpu(wqe->base.wqe_idx); /* wqe_idx must be a valid queue index */ - if (unlikely(wqe_idx >> qp->rq.depth_log2)) { + if (unlikely(wqe_idx >> qp->rq.q.depth_log2)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu invalid\n", qp->qpid, (unsigned long long)wqe_idx); @@ -52,7 +52,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) } /* wqe_idx must indicate a request that is outstanding */ - meta = &qp->rq_meta[wqe_idx]; + meta = &qp->rq.meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(qp->ibqp.device, "flush qp %u recv index %llu not posted\n", @@ -60,7 +60,7 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) return -EIO; } - ionic_queue_consume(&qp->rq); + ionic_queue_consume(&qp->rq.q); memset(wc, 0, sizeof(*wc)); @@ -68,8 +68,8 @@ static int ionic_flush_recv(struct ionic_qp *qp, struct ib_wc *wc) wc->wr_id = meta->wrid; wc->qp = &qp->ibqp; - meta->next = qp->rq_meta_head; - qp->rq_meta_head = meta; + meta->next = qp->rq.meta_head; + qp->rq.meta_head = meta; return 1; } @@ -139,7 +139,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, u32 src_qpn, st_len; u8 op; - if (cqe_qp->rq_flush) + if (cqe_qp->rq.flush) return 0; qp = cqe_qp; @@ -148,38 +148,38 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, /* ignore wqe_idx in case of flush error */ if (ionic_v1_cqe_error(cqe) && st_len == IONIC_STS_WQE_FLUSHED_ERR) { - cqe_qp->rq_flush = true; + cqe_qp->rq.flush = true; cq->flush = true; - list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); + list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); /* posted recvs (if any) flushed by ionic_flush_recv */ return 0; } /* there had better be something in the recv queue to complete */ - if (ionic_queue_empty(&qp->rq)) { + if (ionic_queue_empty(&qp->rq.q)) { ibdev_warn(&dev->ibdev, "qp %u is empty\n", qp->qpid); return -EIO; } wqe_idx = le64_to_cpu(cqe->recv.wqe_idx_timestamp) & IONIC_V1_CQE_WQE_IDX_MASK; /* wqe_idx must be a valid queue index */ - if (unlikely(wqe_idx >> qp->rq.depth_log2)) { + if (unlikely(wqe_idx >> qp->rq.q.depth_log2)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u invalid\n", qp->qpid, wqe_idx); return -EIO; } /* wqe_idx must indicate a request that is outstanding */ - meta = &qp->rq_meta[wqe_idx]; + meta = &qp->rq.meta[wqe_idx]; if (unlikely(meta->next != IONIC_META_POSTED)) { ibdev_warn(&dev->ibdev, "qp %u recv index %u not posted\n", qp->qpid, wqe_idx); return -EIO; } - meta->next = qp->rq_meta_head; - qp->rq_meta_head = meta; + meta->next = qp->rq.meta_head; + qp->rq.meta_head = meta; memset(wc, 0, sizeof(*wc)); @@ -191,9 +191,9 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, wc->vendor_err = st_len; wc->status = ionic_to_ib_status(st_len); - cqe_qp->rq_flush = true; + cqe_qp->rq.flush = true; cq->flush = true; - list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); + list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); ibdev_warn(&dev->ibdev, "qp %d recv cqe with error\n", qp->qpid); @@ -255,7 +255,7 @@ static int ionic_poll_recv(struct ionic_ibdev *dev, struct ionic_cq *cq, wc->port_num = 1; out: - ionic_queue_consume(&qp->rq); + ionic_queue_consume(&qp->rq.q); return 1; } @@ -526,9 +526,15 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev, switch (type) { case IONIC_V1_CQE_TYPE_RECV: - spin_lock(&qp->rq_lock); + if (qp->srq) { + ibdev_dbg(&dev->ibdev, + "srq recv cqe for qp %u\n", qid); + goto cq_next; + } + + spin_lock(&qp->rq.lock); rc = ionic_poll_recv(dev, cq, qp, cqe, wc + npolled); - spin_unlock(&qp->rq_lock); + spin_unlock(&qp->rq.lock); if (rc < 0) goto out; @@ -612,19 +618,19 @@ static int ionic_poll_vcq_cq(struct ionic_ibdev *dev, cq->flush = true; } - list_for_each_entry_safe(qp, qp_next, &cq->flush_rq, cq_flush_rq) { + list_for_each_entry_safe(qp, qp_next, &cq->flush_rq, rq.cq_flush) { if (npolled == nwc) goto out; - spin_lock(&qp->rq_lock); + spin_lock(&qp->rq.lock); rc = ionic_flush_recv_many(qp, wc + npolled, nwc - npolled); - spin_unlock(&qp->rq_lock); + spin_unlock(&qp->rq.lock); if (rc > 0) npolled += rc; if (npolled < nwc) - list_del_init(&qp->cq_flush_rq); + list_del_init(&qp->rq.cq_flush); else cq->flush = true; } @@ -881,7 +887,7 @@ static void ionic_prep_sq_wqe(struct ionic_qp *qp, void *wqe) static void ionic_prep_rq_wqe(struct ionic_qp *qp, void *wqe) { - memset(wqe, 0, 1u << qp->rq.stride_log2); + memset(wqe, 0, 1u << qp->rq.q.stride_log2); } static int ionic_prep_send(struct ionic_qp *qp, @@ -1183,38 +1189,38 @@ static int ionic_prep_recv(struct ionic_qp *qp, s64 signed_len; u32 mval; - wqe = ionic_queue_at_prod(&qp->rq); + wqe = ionic_queue_at_prod(&qp->rq.q); /* if wqe is owned by device, caller can try posting again soon */ if (wqe->base.flags & cpu_to_be16(IONIC_V1_FLAG_FENCE)) return -EAGAIN; - meta = qp->rq_meta_head; + meta = qp->rq.meta_head; if (unlikely(meta == IONIC_META_LAST) || unlikely(meta == IONIC_META_POSTED)) return -EIO; ionic_prep_rq_wqe(qp, wqe); - mval = ionic_v1_recv_wqe_max_sge(qp->rq.stride_log2, qp->rq_spec, + mval = ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2, qp->rq.spec, false); signed_len = ionic_prep_pld(wqe, &wqe->recv.pld, - qp->rq_spec, mval, + qp->rq.spec, mval, wr->sg_list, wr->num_sge); if (signed_len < 0) return signed_len; meta->wrid = wr->wr_id; - wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq_meta); + wqe->base.wqe_idx = cpu_to_le64(meta - qp->rq.meta); wqe->base.num_sge_key = wr->num_sge; /* total length for recv goes in base imm_data_key */ wqe->base.imm_data_key = cpu_to_be32(signed_len); - ionic_queue_produce(&qp->rq); + ionic_queue_produce(&qp->rq.q); - qp->rq_meta_head = meta->next; + qp->rq.meta_head = meta->next; meta->next = IONIC_META_POSTED; return 0; @@ -1311,7 +1317,7 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev, if (!bad) return -EINVAL; - if (!qp->has_rq) { + if (qp->srq) { *bad = wr; return -EINVAL; } @@ -1321,10 +1327,10 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev, return -EINVAL; } - spin_lock_irqsave(&qp->rq_lock, irqflags); + spin_lock_irqsave(&qp->rq.lock, irqflags); while (wr) { - if (ionic_queue_full(&qp->rq)) { + if (ionic_queue_full(&qp->rq.q)) { ibdev_dbg(&dev->ibdev, "queue full"); rc = -ENOMEM; goto out; @@ -1339,32 +1345,32 @@ static int ionic_post_recv_common(struct ionic_ibdev *dev, out: if (!cq) { - spin_unlock_irqrestore(&qp->rq_lock, irqflags); + spin_unlock_irqrestore(&qp->rq.lock, irqflags); goto out_unlocked; } - spin_unlock_irqrestore(&qp->rq_lock, irqflags); + spin_unlock_irqrestore(&qp->rq.lock, irqflags); spin_lock_irqsave(&cq->lock, irqflags); - spin_lock(&qp->rq_lock); + spin_lock(&qp->rq.lock); - if (likely(qp->rq.prod != qp->rq_old_prod)) { + if (likely(qp->rq.q.prod != qp->rq.old_prod)) { /* ring cq doorbell just in time */ - spend = (qp->rq.prod - qp->rq_old_prod) & qp->rq.mask; + spend = (qp->rq.q.prod - qp->rq.old_prod) & qp->rq.q.mask; ionic_reserve_cq(dev, cq, spend); - qp->rq_old_prod = qp->rq.prod; + qp->rq.old_prod = qp->rq.q.prod; ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.rq_qtype, - ionic_queue_dbell_val(&qp->rq)); + ionic_queue_dbell_val(&qp->rq.q)); } - if (qp->rq_flush) { + if (qp->rq.flush) { notify = true; cq->flush = true; - list_move_tail(&qp->cq_flush_rq, &cq->flush_rq); + list_move_tail(&qp->rq.cq_flush, &cq->flush_rq); } - spin_unlock(&qp->rq_lock); + spin_unlock(&qp->rq.lock); spin_unlock_irqrestore(&cq->lock, irqflags); if (notify && vcq->ibcq.comp_handler) diff --git a/drivers/infiniband/hw/ionic/ionic_fw.h b/drivers/infiniband/hw/ionic/ionic_fw.h index ee23062a1762..1db67fcb9a53 100644 --- a/drivers/infiniband/hw/ionic/ionic_fw.h +++ b/drivers/infiniband/hw/ionic/ionic_fw.h @@ -88,6 +88,10 @@ static inline int to_ionic_mr_flags(int access) return flags; } +enum ionic_srq_flags { + IONIC_SRQF_CMB = BIT(0), +}; + enum ionic_qp_flags { /* bits that determine qp access */ IONIC_QPF_REMOTE_WRITE = BIT(0), @@ -105,6 +109,8 @@ enum ionic_qp_flags { IONIC_QPF_SQ_CMB = BIT(13), IONIC_QPF_RQ_CMB = BIT(14), IONIC_QPF_PRIVILEGED = BIT(15), + + IONIC_QP_USER_FLAGS_MASK = GENMASK(31, 16), }; static inline int from_ionic_qp_flags(int flags) @@ -744,11 +750,22 @@ struct ionic_admin_create_cq { __le32 map_count; __le64 dma_addr; __le16 dbid_flags; + __u8 udma_idx; + __le32 resp_buf_len; + __le64 resp_dma_addr; } __packed; #define IONIC_ADMIN_CREATE_CQ_IN_V1_LEN 34 -static_assert(sizeof(struct ionic_admin_create_cq) == - IONIC_ADMIN_CREATE_CQ_IN_V1_LEN); +#define IONIC_ADMIN_CREATE_CQ_IN_V2_LEN 47 +static_assert(sizeof(struct ionic_admin_create_cq) == IONIC_ADMIN_CREATE_CQ_IN_V2_LEN); + +struct ionic_admin_create_cq_resp { + __le32 id; + __u8 udma_idx; +} __packed; + +#define IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN 5 +static_assert(sizeof(struct ionic_admin_create_cq_resp) == IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN); struct ionic_admin_destroy_cq { __le32 cq_id; @@ -844,6 +861,48 @@ struct ionic_admin_query_qp { static_assert(sizeof(struct ionic_admin_query_qp) == IONIC_ADMIN_QUERY_QP_IN_V1_LEN); +struct ionic_admin_create_srq { + __le64 dma_addr; + __le32 map_count; + __le32 pd_id; + __le32 qid; + __le16 dbid; + __le16 low_wqes_limit; + __le16 flags; + __u8 depth_log2; + __u8 stride_log2; + __u8 page_size_log2; + __u8 udma_mask; + __le32 resp_buf_len; + __le64 resp_dma_addr; +} __packed; + +#define IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN 42 +static_assert(sizeof(struct ionic_admin_create_srq) == IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN); + +struct ionic_admin_create_srq_resp { + __le32 id; + __u8 udma_idx; +} __packed; + +#define IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN 5 +static_assert(sizeof(struct ionic_admin_create_srq_resp) == IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN); + +struct ionic_admin_modify_srq { + __le32 qid; + __le16 low_wqes_limit; +} __packed; + +#define IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN 6 +static_assert(sizeof(struct ionic_admin_modify_srq) == IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN); + +struct ionic_admin_destroy_srq { + __le32 qid; +} __packed; + +#define IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN 4 +static_assert(sizeof(struct ionic_admin_destroy_srq) == IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN); + #define ADMIN_WQE_STRIDE 64 #define ADMIN_WQE_HDR_LEN 4 @@ -866,6 +925,9 @@ struct ionic_v1_admin_wqe { struct ionic_admin_destroy_qp destroy_qp; struct ionic_admin_mod_qp mod_qp; struct ionic_admin_query_qp query_qp; + struct ionic_admin_create_srq create_srq; + struct ionic_admin_modify_srq modify_srq; + struct ionic_admin_destroy_srq destroy_srq; } cmd; }; @@ -912,6 +974,10 @@ enum ionic_v1_admin_op { IONIC_V1_ADMIN_DESTROY_AH, IONIC_V1_ADMIN_QP_STATS_HDRS, IONIC_V1_ADMIN_QP_STATS_VALS, + IONIC_V1_ADMIN_CREATE_SRQ = 26, + IONIC_V1_ADMIN_MODIFY_SRQ, + IONIC_V1_ADMIN_QUERY_SRQ, + IONIC_V1_ADMIN_DESTROY_SRQ, IONIC_V1_ADMIN_OPCODES_MAX, }; @@ -947,7 +1013,7 @@ enum ionic_v1_eqe_evt_bits { /* cq error events */ IONIC_V1_EQE_CQ_ERR = 8, - /* qp and srq events */ + /* qp events */ IONIC_V1_EQE_TYPE_QP = 1, /* qp normal events */ IONIC_V1_EQE_SRQ_LEVEL = 0, @@ -958,6 +1024,10 @@ enum ionic_v1_eqe_evt_bits { IONIC_V1_EQE_QP_ERR = 8, IONIC_V1_EQE_QP_ERR_REQUEST = 9, IONIC_V1_EQE_QP_ERR_ACCESS = 10, + /* srq events */ + IONIC_V1_EQE_TYPE_SRQ = 2, + IONIC_V1_EQE_SRQ_LIMIT_REACHED = 0, + IONIC_V1_EQE_SRQ_ERR = 1, }; enum ionic_tfp_csum_profiles { diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index cba7809ec3d9..b4d4544bb7ac 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -73,6 +73,11 @@ static int ionic_query_device(struct ib_device *ibdev, attr->max_ah = dev->lif_cfg.nahs_per_lif; attr->max_fast_reg_page_list_len = dev->lif_cfg.npts_per_lif / 2; attr->max_pkeys = IONIC_PKEY_TBL_LEN; + if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) { + attr->max_srq = dev->lif_cfg.srq_count; + attr->max_srq_wr = IONIC_MAX_SRQ_DEPTH; + attr->max_srq_sge = IONIC_MAX_SRQ_SGES; + } return 0; } @@ -266,6 +271,15 @@ static const struct ib_device_ops ionic_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_mw, ionic_mr, ibmw), }; +static const struct ib_device_ops ionic_srq_ops = { + .create_srq = ionic_create_srq, + .modify_srq = ionic_modify_srq, + .query_srq = ionic_query_srq, + .destroy_srq = ionic_destroy_srq, + + INIT_RDMA_OBJ_SIZE(ib_srq, ionic_srq, ibsrq), +}; + static void ionic_init_resids(struct ionic_ibdev *dev) { ionic_resid_init(&dev->inuse_cqid, dev->lif_cfg.cq_count); @@ -304,6 +318,8 @@ static void ionic_destroy_ibdev(struct ionic_ibdev *dev) xa_destroy(&dev->qp_tbl); WARN_ON(!xa_empty(&dev->cq_tbl)); xa_destroy(&dev->cq_tbl); + WARN_ON(!xa_empty(&dev->srq_tbl)); + xa_destroy(&dev->srq_tbl); ib_dealloc_device(&dev->ibdev); } @@ -320,8 +336,9 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) ionic_fill_lif_cfg(ionic_adev->lif, &dev->lif_cfg); - xa_init_flags(&dev->qp_tbl, GFP_ATOMIC); - xa_init_flags(&dev->cq_tbl, GFP_ATOMIC); + xa_init_flags(&dev->qp_tbl, XA_FLAGS_LOCK_IRQ); + xa_init_flags(&dev->cq_tbl, XA_FLAGS_LOCK_IRQ); + xa_init_flags(&dev->srq_tbl, XA_FLAGS_LOCK_IRQ); ionic_init_resids(dev); @@ -354,6 +371,8 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) goto err_admin; ib_set_device_ops(&dev->ibdev, &ionic_dev_ops); + if (dev->lif_cfg.srq_count && ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_SRQ)) + ib_set_device_ops(&dev->ibdev, &ionic_srq_ops); ionic_stats_init(dev); @@ -372,6 +391,7 @@ static struct ionic_ibdev *ionic_create_ibdev(struct ionic_aux_dev *ionic_adev) ionic_destroy_resids(dev); xa_destroy(&dev->qp_tbl); xa_destroy(&dev->cq_tbl); + xa_destroy(&dev->srq_tbl); ib_dealloc_device(&dev->ibdev); return ERR_PTR(rc); diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h index 32b6a8a45fa2..f8d580dd4d95 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.h +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -36,6 +36,10 @@ #define IONIC_SQCMB_ORDER 5 #define IONIC_RQCMB_ORDER 0 +#define IONIC_MAX_SRQ_SGES 2 +#define IONIC_MAX_SRQ_LIMIT 0xffff +#define IONIC_MAX_SRQ_DEPTH 0xffff + #define IONIC_META_LAST ((void *)1ul) #define IONIC_META_POSTED ((void *)2ul) @@ -88,6 +92,7 @@ struct ionic_ibdev { struct xarray qp_tbl; struct xarray cq_tbl; + struct xarray srq_tbl; struct ionic_resid_bits inuse_dbid; struct ionic_resid_bits inuse_pdid; @@ -241,6 +246,37 @@ struct ionic_rq_meta { u64 wrid; }; +struct ionic_rq { + struct ionic_queue q; + u32 qid; + u32 cqid; + struct list_head cq_flush; + spinlock_t lock; /* for posting and polling */ + + phys_addr_t cmb_addr; + struct rdma_user_mmap_entry *mmap_cmb; + + struct ionic_rq_meta *meta; + struct ionic_rq_meta *meta_head; + struct ib_umem *umem; + + int spec; + int cmb_order; + u32 cmb_pgid; + u16 old_prod; + u8 cmb; + bool flush; +}; + +struct ionic_srq { + struct ib_srq ibsrq; + struct ionic_rq rq; + struct kref kref; + struct completion rel_comp; + u16 srq_limit; + u8 udma_idx; +}; + struct ionic_qp { struct ib_qp ibqp; enum ib_qp_state state; @@ -248,18 +284,15 @@ struct ionic_qp { u32 qpid; u32 ahid; u32 sq_cqid; - u32 rq_cqid; u8 udma_idx; u8 has_ah:1; u8 has_sq:1; - u8 has_rq:1; u8 sig_all:1; struct list_head qp_list_counter; struct list_head cq_poll_sq; struct list_head cq_flush_sq; - struct list_head cq_flush_rq; struct list_head ibkill_flush_ent; spinlock_t sq_lock; /* for posting and polling */ @@ -274,14 +307,7 @@ struct ionic_qp { bool sq_flush; bool sq_flush_rcvd; - spinlock_t rq_lock; /* for posting and polling */ - struct ionic_queue rq; - struct ionic_rq_meta *rq_meta; - struct ionic_rq_meta *rq_meta_head; - int rq_spec; - u16 rq_old_prod; - u8 rq_cmb; - bool rq_flush; + struct ionic_rq rq; struct kref qp_kref; struct completion qp_rel_comp; @@ -295,16 +321,10 @@ struct ionic_qp { struct ib_umem *sq_umem; - int rq_cmb_order; - u32 rq_cmb_pgid; - phys_addr_t rq_cmb_addr; - struct rdma_user_mmap_entry *mmap_rq_cmb; - - struct ib_umem *rq_umem; - int dcqcn_profile; struct ib_ud_header *hdr; + struct ionic_srq *srq; }; struct ionic_ah { @@ -399,6 +419,11 @@ static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp) return container_of(ibqp, struct ionic_qp, ibqp); } +static inline struct ionic_srq *to_ionic_srq(struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct ionic_srq, ibsrq); +} + static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah) { return container_of(ibah, struct ionic_ah, ibah); @@ -438,6 +463,19 @@ static inline void ionic_cq_complete(struct kref *kref) complete(&cq->cq_rel_comp); } +static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev, + enum ionic_lif_rdma_alloc_qid qtype) +{ + return dev->lif_cfg.alloc_qid_cap & qtype; +} + +static inline void ionic_srq_complete(struct kref *kref) +{ + struct ionic_srq *srq = container_of(kref, struct ionic_srq, kref); + + complete(&srq->rel_comp); +} + /* ionic_admin.c */ extern struct workqueue_struct *ionic_evt_workq; void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr); @@ -457,8 +495,11 @@ int ionic_create_cq_common(struct ionic_vcq *vcq, struct ionic_ctx *ctx, struct ib_udata *udata, struct ionic_qdesc *req_cq, - __u32 *resp_cqid, int udma_idx); +int ionic_post_create_cq_cmd(struct ionic_cq *cq, + struct ib_udata *udata, + __u32 *resp_cqid); +void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq); void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq); void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp); void ionic_notify_flush_cq(struct ionic_cq *cq); @@ -498,6 +539,12 @@ int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask, struct ib_qp_init_attr *init_attr); int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); +int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr, + struct ib_udata *udata); +int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata); +int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr); +int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); /* ionic_datapath.c */ int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c index 8e48c00d4959..66cee57442e7 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c @@ -75,6 +75,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) cfg->eq_count = lif->ionic->neqs_per_lif; cfg->cq_count = le32_to_cpu(ident->rdma.cq_qtype.qid_count); cfg->qp_count = le32_to_cpu(ident->rdma.sq_qtype.qid_count); + cfg->srq_count = le32_to_cpu(ident->rdma.srq_qtype.qid_count); cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif); cfg->aq_qtype = ident->rdma.aq_qtype.qtype; @@ -86,12 +87,14 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) cfg->udma_count = 2; cfg->max_stride = ident->rdma.max_stride; + cfg->rcq_sign_bit = ident->rdma.rcq_sign_bit; cfg->expdb_mask = ionic_get_expdb(lif); cfg->sq_expdb = !!(lif->qtype_info[IONIC_QTYPE_TXQ].features & IONIC_QIDENT_F_EXPDB); cfg->rq_expdb = !!(lif->qtype_info[IONIC_QTYPE_RXQ].features & IONIC_QIDENT_F_EXPDB); + cfg->alloc_qid_cap = ident->rdma.alloc_qid_cap; } struct net_device *ionic_lif_netdev(struct ionic_lif *lif) diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h index 2b29e646c193..bc14fc0921da 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.h +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.h @@ -38,6 +38,7 @@ struct ionic_lif_cfg { int eq_count; int cq_count; int qp_count; + int srq_count; u16 stats_type; u8 aq_qtype; @@ -57,6 +58,8 @@ struct ionic_lif_cfg { bool sq_expdb; bool rq_expdb; u8 expdb_mask; + u8 rcq_sign_bit; + u8 alloc_qid_cap; }; void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg); diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c index 335ae3c82e17..1c2ad5773b9f 100644 --- a/drivers/infiniband/hw/irdma/ctrl.c +++ b/drivers/infiniband/hw/irdma/ctrl.c @@ -1689,11 +1689,9 @@ static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev, * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp * @qp: sc qp struct * @info: fast mr info - * @post_sq: flag for cqp db to ring */ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, - struct irdma_fast_reg_stag_info *info, - bool post_sq) + struct irdma_fast_reg_stag_info *info) { u64 temp, hdr; __le64 *wqe; @@ -1723,15 +1721,13 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, info->wr_id, wqe_idx, &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); - temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? - (uintptr_t)info->va : info->fbo; - set_64bit_val(wqe, 0, temp); + set_64bit_val(wqe, 0, info->va); temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI, info->first_pm_pbl_index >> 16); set_64bit_val(wqe, 8, FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) | - FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa)); + FIELD_PREP(IRDMAQPSQ_PBLADDR, info->reg_addr_pa >> IRDMA_HW_PAGE_SHIFT)); set_64bit_val(wqe, 16, info->total_len | FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index)); @@ -1742,7 +1738,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) | FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) | FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) | - FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) | + IRDMAQPSQ_VABASEDTO | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | @@ -1755,8 +1751,7 @@ int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe, IRDMA_QP_WQE_MIN_SIZE, false); - if (post_sq) - irdma_uk_qp_post_wr(&qp->qp_uk); + irdma_uk_qp_post_wr(&qp->qp_uk); return 0; } @@ -2625,11 +2620,9 @@ static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp, * @qp: sc qp * @info: dlush information * @scratch: u64 saved to be used during cqp completion - * @post_sq: flag for cqp db to ring */ int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, - struct irdma_qp_flush_info *info, u64 scratch, - bool post_sq) + struct irdma_qp_flush_info *info, u64 scratch) { u64 temp = 0; __le64 *wqe; @@ -2697,8 +2690,7 @@ int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false); - if (post_sq) - irdma_sc_cqp_post_sq(cqp); + irdma_sc_cqp_post_sq(cqp); return 0; } @@ -5331,7 +5323,11 @@ int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, */ static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp) { - return IRDMA_RING_FULL_ERR(cqp->sq_ring); + u64 in_flight = cqp->requested_ops - + atomic64_read(&cqp->completed_ops); + + return IRDMA_RING_FULL_ERR(cqp->sq_ring) || + in_flight >= (cqp->sq_size - 1); } /** @@ -6114,8 +6110,7 @@ static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev, case IRDMA_OP_QP_FLUSH_WQES: status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp, &pcmdinfo->in.u.qp_flush_wqes.info, - pcmdinfo->in.u.qp_flush_wqes.scratch, - pcmdinfo->post_sq); + pcmdinfo->in.u.qp_flush_wqes.scratch); break; case IRDMA_OP_GEN_AE: status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp, diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index c345cc654256..fbb8fb47d0d1 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -265,6 +265,7 @@ static void irdma_process_ae_def_cmpl(struct irdma_pci_f *rf, irdma_sc_cqp_def_cmpl_ae_handler(&rf->sc_dev, info, false, &scratch, &sw_def_info); } + irdma_process_bh(&rf->sc_dev); } /** @@ -1252,12 +1253,21 @@ static int irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq, } msix_vec->ceq_id = ceq_id; - if (rf->sc_dev.privileged) + if (rf->sc_dev.privileged) { rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx, true); - else + } else { status = irdma_vchnl_req_ceq_vec_map(&rf->sc_dev, ceq_id, msix_vec->idx); + if (!status) + return 0; + + if (rf->msix_shared && !ceq_id) + irdma_destroy_irq(rf, msix_vec, rf); + else + irdma_destroy_irq(rf, msix_vec, iwceq); + } + return status; } @@ -2742,7 +2752,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, hw_info = &cqp_request->info.in.u.qp_flush_wqes.info; memcpy(hw_info, info, sizeof(*hw_info)); cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES; - cqp_info->post_sq = 1; cqp_info->in.u.qp_flush_wqes.qp = qp; cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)cqp_request; status = irdma_handle_cqp_op(rf, cqp_request); @@ -2789,7 +2798,6 @@ int irdma_hw_flush_wqes(struct irdma_pci_f *rf, struct irdma_sc_qp *qp, hw_info = &new_req->info.in.u.qp_flush_wqes.info; memcpy(hw_info, info, sizeof(*hw_info)); cqp_info->cqp_cmd = IRDMA_OP_QP_FLUSH_WQES; - cqp_info->post_sq = 1; cqp_info->in.u.qp_flush_wqes.qp = qp; cqp_info->in.u.qp_flush_wqes.scratch = (uintptr_t)new_req; diff --git a/drivers/infiniband/hw/irdma/icrdma_if.c b/drivers/infiniband/hw/irdma/icrdma_if.c index 4b451d8482a4..f405ef761f84 100644 --- a/drivers/infiniband/hw/irdma/icrdma_if.c +++ b/drivers/infiniband/hw/irdma/icrdma_if.c @@ -318,14 +318,15 @@ static void icrdma_remove(struct auxiliary_device *aux_dev) container_of(aux_dev, struct iidc_rdma_core_auxiliary_dev, adev); struct iidc_rdma_core_dev_info *cdev_info = idc_adev->cdev_info; struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev); - u8 rdma_ver = iwdev->rf->rdma_ver; + struct irdma_pci_f *rf = iwdev->rf; + u8 rdma_ver = rf->rdma_ver; ice_rdma_update_vsi_filter(cdev_info, iwdev->vsi_num, false); irdma_ib_unregister_device(iwdev); - icrdma_deinit_interrupts(iwdev->rf, cdev_info); - mutex_destroy(&iwdev->rf->ah_tbl_lock); + icrdma_deinit_interrupts(rf, cdev_info); + mutex_destroy(&rf->ah_tbl_lock); - kfree(iwdev->rf); + kfree(rf); pr_debug("INIT: Gen[%d] func[%d] device remove success\n", rdma_ver, PCI_FUNC(cdev_info->pdev->devfn)); diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c index 95957d52883d..106915b51e7c 100644 --- a/drivers/infiniband/hw/irdma/main.c +++ b/drivers/infiniband/hw/irdma/main.c @@ -80,6 +80,8 @@ static int ig3rdma_vport_probe(struct auxiliary_device *aux_dev, return -ENOMEM; } iwdev = ib_alloc_device(irdma_device, ibdev); + if (!iwdev) + return -ENOMEM; /* Fill iwdev info */ iwdev->is_vport = true; iwdev->rf = rf; diff --git a/drivers/infiniband/hw/irdma/pble.c b/drivers/infiniband/hw/irdma/pble.c index 28dfad7f940c..0c2c5e96a05a 100644 --- a/drivers/infiniband/hw/irdma/pble.c +++ b/drivers/infiniband/hw/irdma/pble.c @@ -330,7 +330,7 @@ static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc, if (root->addr) irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo); - kfree(lvl2->leafmem.va); + kvfree(lvl2->leafmem.va); lvl2->leaf = NULL; } @@ -358,7 +358,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, lvl2->leaf_cnt = total; lvl2->leafmem.size = (sizeof(*leaf) * total); - lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL); + lvl2->leafmem.va = kvzalloc(lvl2->leafmem.size, GFP_KERNEL); if (!lvl2->leafmem.va) return -ENOMEM; @@ -367,7 +367,7 @@ static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo, total << 3, &root->addr, &fpm_addr); if (ret_code) { - kfree(lvl2->leafmem.va); + kvfree(lvl2->leafmem.va); lvl2->leaf = NULL; return -ENOMEM; } @@ -482,7 +482,7 @@ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl); /* if level1_only, only go through it once */ - if (!status || lvl) + if (!status || lvl == PBLE_LEVEL_1) break; } diff --git a/drivers/infiniband/hw/irdma/protos.h b/drivers/infiniband/hw/irdma/protos.h index 324cfbf21764..561a0287b945 100644 --- a/drivers/infiniband/hw/irdma/protos.h +++ b/drivers/infiniband/hw/irdma/protos.h @@ -19,8 +19,7 @@ void irdma_sc_rt_init(struct irdma_sc_dev *dev); void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp); __le64 *irdma_sc_cqp_get_next_send_wqe(struct irdma_sc_cqp *cqp, u64 scratch); int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp, - struct irdma_fast_reg_stag_info *info, - bool post_sq); + struct irdma_fast_reg_stag_info *info); /* HMC/FPM functions */ int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id); /* stats misc */ diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h index 5557d9338796..8b4d2c0b283f 100644 --- a/drivers/infiniband/hw/irdma/type.h +++ b/drivers/infiniband/hw/irdma/type.h @@ -1173,13 +1173,11 @@ struct irdma_reg_ns_stag_info { struct irdma_fast_reg_stag_info { u64 wr_id; u64 reg_addr_pa; - u64 fbo; - void *va; + u64 va; u64 total_len; u32 page_size; u32 chunk_size; u32 first_pm_pbl_index; - enum irdma_addressing_type addr_type; irdma_stag_index stag_idx; u16 access_rights; u32 pd_id; @@ -1383,8 +1381,7 @@ int irdma_sc_qp_create(struct irdma_sc_qp *qp, int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch, bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq); int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp, - struct irdma_qp_flush_info *info, u64 scratch, - bool post_sq); + struct irdma_qp_flush_info *info, u64 scratch); int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info); int irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info, u64 scratch, diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 5d71300da0ff..c91c55917de9 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -1576,9 +1576,17 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, issue_modify_qp = 1; iwqp->suspend_pending = true; break; + case IB_QPS_RESET: + /* + * A QP that was never moved to INIT has nothing to + * flush, so skip the hardware modify. Moving it to + * ERROR would make the following INIT fail. + */ + if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) + break; + fallthrough; case IB_QPS_SQE: case IB_QPS_ERR: - case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { iwqp->ibqp_state = attr->qp_state; spin_unlock_irqrestore(&iwqp->lock, flags); @@ -2909,42 +2917,6 @@ static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) return true; } -/** - * irdma_check_mr_contiguous - check if MR is physically contiguous - * @palloc: pbl allocation struct - * @pg_size: page size - */ -static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, - u32 pg_size) -{ - struct irdma_pble_level2 *lvl2 = &palloc->level2; - struct irdma_pble_info *leaf = lvl2->leaf; - u64 *arr = NULL; - u64 *start_addr = NULL; - int i; - bool ret; - - if (palloc->level == PBLE_LEVEL_1) { - arr = palloc->level1.addr; - ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, - pg_size); - return ret; - } - - start_addr = leaf->addr; - - for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { - arr = leaf->addr; - if ((*start_addr + ((u64)i * pg_size * PBLE_PER_PAGE)) != *arr) - return false; - ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); - if (!ret) - return false; - } - - return true; -} - /** * irdma_setup_pbles - copy user pg address to pble's * @rf: RDMA PCI function @@ -2981,9 +2953,6 @@ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, irdma_copy_user_pgaddrs(iwmr, pbl, pbl_len, level); - if (lvl) - iwmr->pgaddrmem[0] = *pbl; - return 0; } @@ -3405,24 +3374,33 @@ static int irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access, { struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct ib_umem *region = iwmr->region; u32 stag = 0; - u8 lvl; int err; - lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; iwmr->access = access; - err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); - if (err) - return err; - - if (lvl) { - err = irdma_check_mr_contiguous(&iwpbl->pble_alloc, + if (ib_umem_is_contiguous(region)) { + /* HW needs a valid page size even without PBLEs, so use 4K. It + * behaves like a PBLE-backed region where the PBA must be page + * aligned. HW takes the page offset from the lower 12 VA bits. + */ + iwmr->page_size = SZ_4K; + iwmr->pgaddrmem[0] = ALIGN_DOWN(ib_umem_start_dma_addr(region), iwmr->page_size); - if (err) { - irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); - iwpbl->pbl_allocated = false; - } + } else { + iwmr->page_size = ib_umem_find_best_pgsz(region, + iwdev->rf->sc_dev.hw_attrs.page_size_cap, + iwmr->ibmr.iova); + if (unlikely(!iwmr->page_size)) + return -EOPNOTSUPP; + + iwmr->page_cnt = ib_umem_num_dma_blocks(region, + iwmr->page_size); + err = irdma_setup_pbles(iwdev->rf, iwmr, + PBLE_LEVEL_1 | PBLE_LEVEL_2); + if (err) + return err; } if (create_stag) { @@ -3458,10 +3436,8 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, struct ib_pd *pd, u64 virt, enum irdma_memreg_type reg_type) { - struct irdma_device *iwdev = to_iwdev(pd->device); struct irdma_pbl *iwpbl; struct irdma_mr *iwmr; - unsigned long pgsz_bitmap; iwmr = kzalloc_obj(*iwmr); if (!iwmr) @@ -3475,18 +3451,21 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, iwmr->ibmr.device = pd->device; iwmr->ibmr.iova = virt; iwmr->type = reg_type; + iwmr->len = region->length; + iwpbl->user_base = virt; - pgsz_bitmap = (reg_type == IRDMA_MEMREG_TYPE_MEM) ? - iwdev->rf->sc_dev.hw_attrs.page_size_cap : SZ_4K; + /* Page size and count are calculated in irdma_reg_user_mr_type_mem() + * for MEM type MRs. + */ + if (reg_type == IRDMA_MEMREG_TYPE_MEM) + return iwmr; - iwmr->page_size = ib_umem_find_best_pgsz(region, pgsz_bitmap, virt); + iwmr->page_size = ib_umem_find_best_pgsz(region, SZ_4K, virt); if (unlikely(!iwmr->page_size)) { kfree(iwmr); return ERR_PTR(-EOPNOTSUPP); } - iwmr->len = region->length; - iwpbl->user_base = virt; iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); return iwmr; @@ -3825,7 +3804,6 @@ static int irdma_hwdereg_mr(struct ib_mr *ib_mr) static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, u64 virt) { - struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct ib_pd *pd = iwmr->ibmr.pd; struct ib_umem *region; @@ -3838,17 +3816,8 @@ static int irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len, iwmr->region = region; iwmr->ibmr.iova = virt; iwmr->ibmr.pd = pd; - iwmr->page_size = ib_umem_find_best_pgsz(region, - iwdev->rf->sc_dev.hw_attrs.page_size_cap, - virt); - if (unlikely(!iwmr->page_size)) { - err = -EOPNOTSUPP; - goto err; - } - iwmr->len = region->length; iwpbl->user_base = virt; - iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false); if (err) @@ -4276,16 +4245,27 @@ static int irdma_post_send(struct ib_qp *ibqp, stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; stag_info.page_size = reg_wr(ib_wr)->mr->page_size; stag_info.wr_id = ib_wr->wr_id; - stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; - stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; + stag_info.va = iwmr->ibmr.iova; stag_info.total_len = iwmr->ibmr.length; - stag_info.reg_addr_pa = *palloc->level1.addr; - stag_info.first_pm_pbl_index = palloc->level1.idx; stag_info.local_fence = true; - if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) - stag_info.chunk_size = 1; - err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, - true); + if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) { + if (palloc->level == PBLE_LEVEL_1) { + stag_info.first_pm_pbl_index = + palloc->level1.idx; + stag_info.chunk_size = 1; + } else { + stag_info.first_pm_pbl_index = + palloc->level2.root.idx; + stag_info.chunk_size = 3; + } + } else { + stag_info.reg_addr_pa = + (palloc->level == PBLE_LEVEL_1) ? + *palloc->level1.addr : + *palloc->level2.leaf->addr; + } + err = irdma_sc_mr_fast_register(&iwqp->sc_qp, + &stag_info); break; } default: diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index d4e5e3f91268..843e2c64b32b 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -5,6 +5,58 @@ #include "mana_ib.h" +static enum ib_wc_status vendor_error_to_wc_error(uint32_t vendor_error) +{ + switch (vendor_error) { + case VENDOR_ERR_OK: + return IB_WC_SUCCESS; + case VENDOR_ERR_RX_PKT_LEN: + case VENDOR_ERR_RX_MSG_LEN_OVFL: + return IB_WC_LOC_LEN_ERR; + case VENDOR_ERR_TX_GDMA_CORRUPTED_WQE: + case VENDOR_ERR_TX_PCIE_WQE: + case VENDOR_ERR_TX_PCIE_MSG: + case VENDOR_ERR_RX_MALFORMED_WQE: + case VENDOR_ERR_TX_GDMA_INVALID_STATE: + case VENDOR_ERR_TX_MISBEHAVING_CLIENT: + case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE: + case VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD: + case VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED: + case VENDOR_ERR_TX_RDMA_WQE_LEN_ERR: + case VENDOR_ERR_TX_RDMA_MTU_ERR: + return IB_WC_LOC_QP_OP_ERR; + case VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION: + case VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE: + case VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR: + case VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION: + case VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE: + case VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR: + case VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE: + case VENDOR_ERR_RX_ATB_SGE_MISSCONFIG: + return IB_WC_LOC_PROT_ERR; + case VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT: + case VENDOR_ERR_RX_GFID: + return IB_WC_LOC_ACCESS_ERR; + case VENDOR_ERR_RX_MISBEHAVING_CLIENT: + case VENDOR_ERR_RX_CLIENT_ID: + case VENDOR_ERR_RX_PCIE: + case VENDOR_ERR_RX_NO_AVAIL_WQE: + case VENDOR_ERR_RX_ATB_WQE_MISCONFIG: + case VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT: + case VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE: + case VENDOR_ERR_TX_RDMA_INVALID_STATE: + case VENDOR_ERR_TX_RDMA_INVALID_NPT: + case VENDOR_ERR_TX_RDMA_INVALID_SGID: + case VENDOR_ERR_TX_RDMA_VFID_MISMATCH: + return IB_WC_FATAL_ERR; + case VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE: + case VENDOR_ERR_SW_FLUSHED: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } +} + int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct uverbs_attr_bundle *attrs) { @@ -40,7 +92,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->cqe = attr->cqe; err = mana_ib_create_queue(mdev, ucmd.buf_addr, cq->cqe * COMP_ENTRY_SIZE, - &cq->queue); + &cq->queue, true); if (err) { ibdev_dbg(ibdev, "Failed to create queue for create cq, %d\n", err); return err; @@ -64,6 +116,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, doorbell = mdev->gdma_dev->doorbell; } + ibcq->cqe = cq->cqe; + cq->poll_credit = (cq->cqe << (GDMA_CQE_OWNER_BITS - 1)) - 1; + if (is_rnic_cq) { err = mana_ib_gd_create_cq(mdev, cq, doorbell); if (err) { @@ -86,8 +141,8 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, } spin_lock_init(&cq->cq_lock); - INIT_LIST_HEAD(&cq->list_send_qp); - INIT_LIST_HEAD(&cq->list_recv_qp); + INIT_LIST_HEAD(&cq->send_err_qp_list); + INIT_LIST_HEAD(&cq->recv_err_qp_list); return 0; @@ -174,171 +229,290 @@ void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) gc->cq_table[cq->queue.id] = NULL; } +static inline bool gdma_cq_idx_produced(struct gdma_queue *gdma_cq, uint32_t idx) +{ + struct gdma_mem_info *gmi = &gdma_cq->mem_info; + u32 num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE; + u32 expected_bits = (idx / num_cqe) & GDMA_CQE_OWNER_MASK; + u32 offset = (idx % num_cqe) * GDMA_CQE_SIZE; + struct gdma_cqe *cqe; + + if (gmi->nr_pages) + cqe = gmi->pages_va[offset / PAGE_SIZE] + + (offset & (PAGE_SIZE - 1)); + else + cqe = gdma_cq->queue_mem_ptr + offset; + + return cqe->cqe_info.owner_bits == expected_bits; +} + +static inline void mana_ib_cq_doorbell(struct mana_ib_cq *cq, uint8_t arm) +{ + struct mana_ib_dev *mdev = container_of(cq->ibcq.device, struct mana_ib_dev, ib_dev); + struct gdma_queue *gdma_cq = cq->queue.kmem; + u32 num_cqe, max_credit, idx; + + num_cqe = gdma_cq->queue_size / GDMA_CQE_SIZE; + max_credit = num_cqe << (GDMA_CQE_OWNER_BITS - 1); + idx = gdma_cq->head; + + if (cq->poll_credit >= max_credit) { + if (gdma_cq_idx_produced(gdma_cq, idx + cq->poll_credit - max_credit)) + cq->poll_credit++; + else + return; + } else { + /* Set index of already polled CQE for unarm */ + cq->poll_credit = max_credit - (arm ? 0 : 1); + } + + idx += (cq->poll_credit - max_credit); + idx %= (num_cqe << GDMA_CQE_OWNER_BITS); + + mana_gd_wq_ring_doorbell_ext(mdev_to_gc(mdev), gdma_cq, idx, arm, 0); +} + int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); struct gdma_queue *gdma_cq = cq->queue.kmem; + unsigned long irq_flags; if (!gdma_cq) return -EINVAL; - mana_gd_ring_cq(gdma_cq, SET_ARM_BIT); + spin_lock_irqsave(&cq->cq_lock, irq_flags); + mana_ib_cq_doorbell(cq, SET_ARM_BIT); + spin_unlock_irqrestore(&cq->cq_lock, irq_flags); + return 0; } -static inline void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) +struct mana_cq_poll { + struct ib_wc *wc; + int budget; + int produced; +}; + +static struct ib_wc *mana_fill_wc(struct mana_ib_qp *qp, + struct mana_cq_poll *poll, + const struct shadow_wqe_header *wqe, + enum ib_wc_opcode opcode, u32 vendor_error) { - struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; - struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; - struct ud_sq_shadow_wqe *shadow_wqe; + struct ib_wc *wc = &poll->wc[poll->produced++]; - shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq); - if (!shadow_wqe) - return; + memset(wc, 0, sizeof(*wc)); + wc->wr_id = wqe->wr_id; + wc->status = vendor_error_to_wc_error(vendor_error); + wc->opcode = opcode; + wc->vendor_err = vendor_error; + wc->qp = &qp->ibqp; - shadow_wqe->header.error_code = rdma_cqe->ud_send.vendor_error; - - wq->tail += shadow_wqe->header.posted_wqe_size; - shadow_queue_advance_next_to_complete(&qp->shadow_sq); + return wc; } -static inline void handle_ud_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe) +static void mana_complete_send(struct mana_ib_qp *qp, + struct mana_cq_poll *poll, u32 vendor_error) { - struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; - struct gdma_queue *wq = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; - struct ud_rq_shadow_wqe *shadow_wqe; + struct shadow_queue *shadow = &qp->shadow_sq; + struct shadow_wqe_header *wqe = shadow_queue_get_next_to_consume(shadow); + struct gdma_queue *queue; - shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_rq); - if (!shadow_wqe) + if (!wqe) return; - shadow_wqe->byte_len = rdma_cqe->ud_recv.msg_len; - shadow_wqe->src_qpn = rdma_cqe->ud_recv.src_qpn; - shadow_wqe->header.error_code = IB_WC_SUCCESS; + if (vendor_error || !(wqe->flags & MANA_WQ_NO_SIGNAL_WC)) + mana_fill_wc(qp, poll, wqe, wqe->send_opcode, vendor_error); - wq->tail += shadow_wqe->header.posted_wqe_size; - shadow_queue_advance_next_to_complete(&qp->shadow_rq); + queue = mana_qp_get_sq(qp)->kmem; + queue->tail += wqe->wqe_size_in_bu; + shadow_queue_advance_consumer(shadow); } -static void mana_handle_cqe(struct mana_ib_dev *mdev, struct gdma_comp *cqe) +static void handle_ud_sq_cqe(struct mana_ib_qp *qp, struct mana_rdma_cqe *rdma_cqe, + struct mana_cq_poll *poll) { + u32 offset = rdma_cqe->ud_send.tx_wqe_offset & MANA_WQE_OFFSET_MASK; + struct shadow_queue *shadow = &qp->shadow_sq; + struct shadow_wqe_header *wqe; + u64 idx = shadow->cons_idx; + u32 to_complete = 0; + u64 prod_idx; + + /* Pair with posting's release of the initialized shadow entries. */ + prod_idx = smp_load_acquire(&shadow->prod_idx); + /* Find the target before retiring any entries: the CQE may be stale. */ + for (; idx != prod_idx; idx++) { + wqe = shadow_queue_get_element(shadow, idx); + to_complete++; + if (wqe->wqe_offset_or_psn == offset) + break; + if (!(wqe->flags & MANA_WQ_NO_SIGNAL_WC)) + return; + } + if (idx == prod_idx) + return; + + for (; to_complete; to_complete--) + mana_complete_send(qp, poll, VENDOR_ERR_OK); +} + +static void handle_rq_cqe(struct mana_ib_qp *qp, struct gdma_comp *cqe, + struct mana_cq_poll *poll) +{ + struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; + u32 offset = rdma_cqe->ud_recv.rx_wqe_offset / GDMA_WQE_BU_SIZE; + struct mana_ib_queue *rq = mana_qp_get_rq(qp); + struct gdma_queue *wq = rq->kmem; + struct shadow_wqe_header *wqe; + struct ib_wc *wc; + + wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq); + if (!wqe || wqe->wqe_offset_or_psn != (offset & MANA_WQE_OFFSET_MASK)) + return; + + wc = mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_OK); + switch (rdma_cqe->cqe_type) { + case CQE_TYPE_UD_SEND_IMM: + wc->ex.imm_data = cpu_to_be32(rdma_cqe->ud_recv.imm_data); + wc->wc_flags |= IB_WC_WITH_IMM; + fallthrough; + case CQE_TYPE_UD_SEND: + wc->byte_len = rdma_cqe->ud_recv.msg_len; + wc->src_qp = rdma_cqe->ud_recv.src_qpn; + wc->wc_flags |= IB_WC_GRH; + break; + default: + break; + } + + wq->tail += wqe->wqe_size_in_bu; + shadow_queue_advance_consumer(&qp->shadow_rq); +} + +static bool mana_handle_cqe(struct mana_ib_cq *cq, struct mana_ib_dev *mdev, + struct mana_cq_poll *poll) +{ + struct gdma_comp *cqe = &cq->pending_cqe; + struct mana_rdma_cqe *rdma_cqe = (struct mana_rdma_cqe *)cqe->cqe_data; struct mana_ib_qp *qp = mana_get_qp_ref(mdev, cqe->wq_num, cqe->is_sq); if (!qp) + return true; + + switch (rdma_cqe->cqe_type) { + case CQE_TYPE_UD_SEND: + if (cqe->is_sq) { + handle_ud_sq_cqe(qp, rdma_cqe, poll); + break; + } + fallthrough; + case CQE_TYPE_UD_SEND_IMM: + handle_rq_cqe(qp, cqe, poll); + break; + default: + ibdev_warn_ratelimited(qp->ibqp.device, "Unexpected CQE type %u\n", + rdma_cqe->cqe_type); + break; + } + mana_put_qp_ref(qp); + return true; +} + +static void mana_flush_completions(struct mana_ib_cq *cq, struct mana_cq_poll *poll) +{ + struct shadow_wqe_header *wqe; + struct mana_ib_qp *qp; + + if (poll->produced >= poll->budget) return; - if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_UD) { - if (cqe->is_sq) - handle_ud_sq_cqe(qp, cqe); - else - handle_ud_rq_cqe(qp, cqe); + list_for_each_entry(qp, &cq->send_err_qp_list, send_err_node) { + while (poll->produced < poll->budget && + shadow_queue_get_next_to_consume(&qp->shadow_sq)) + mana_complete_send(qp, poll, VENDOR_ERR_SW_FLUSHED); + if (poll->produced == poll->budget) + return; } - mana_put_qp_ref(qp); -} - -static void fill_verbs_from_shadow_wqe(struct mana_ib_qp *qp, struct ib_wc *wc, - const struct shadow_wqe_header *shadow_wqe) -{ - const struct ud_rq_shadow_wqe *ud_wqe = (const struct ud_rq_shadow_wqe *)shadow_wqe; - - wc->wr_id = shadow_wqe->wr_id; - wc->status = shadow_wqe->error_code; - wc->opcode = shadow_wqe->opcode; - wc->vendor_err = shadow_wqe->error_code; - wc->wc_flags = 0; - wc->qp = &qp->ibqp; - wc->pkey_index = 0; - - if (shadow_wqe->opcode == IB_WC_RECV) { - wc->byte_len = ud_wqe->byte_len; - wc->src_qp = ud_wqe->src_qpn; - wc->wc_flags |= IB_WC_GRH; - } -} - -static int mana_process_completions(struct mana_ib_cq *cq, int nwc, struct ib_wc *wc) -{ - struct shadow_wqe_header *shadow_wqe; - struct mana_ib_qp *qp; - int wc_index = 0; - - /* process send shadow queue completions */ - list_for_each_entry(qp, &cq->list_send_qp, cq_send_list) { - while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_sq)) - != NULL) { - if (wc_index >= nwc) - goto out; - - fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); - shadow_queue_advance_consumer(&qp->shadow_sq); - wc_index++; - } - } - - /* process recv shadow queue completions */ - list_for_each_entry(qp, &cq->list_recv_qp, cq_recv_list) { - while ((shadow_wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq)) - != NULL) { - if (wc_index >= nwc) - goto out; - - fill_verbs_from_shadow_wqe(qp, &wc[wc_index], shadow_wqe); + list_for_each_entry(qp, &cq->recv_err_qp_list, recv_err_node) { + while (poll->produced < poll->budget && + (wqe = shadow_queue_get_next_to_consume(&qp->shadow_rq))) { + mana_fill_wc(qp, poll, wqe, IB_WC_RECV, VENDOR_ERR_SW_FLUSHED); shadow_queue_advance_consumer(&qp->shadow_rq); - wc_index++; } + if (poll->produced == poll->budget) + return; } - -out: - return wc_index; } -void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) +static void mana_drain_gsi_sq(struct mana_ib_qp *qp) { - struct mana_ib_qp *qp = mana_get_qp_ref(mdev, MANA_GSI_QPN, false); - struct ud_sq_shadow_wqe *shadow_wqe; - struct mana_ib_cq *cq; + struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); unsigned long flags; - if (!qp) - return; - - cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); - spin_lock_irqsave(&cq->cq_lock, flags); - while ((shadow_wqe = shadow_queue_get_next_to_complete(&qp->shadow_sq)) - != NULL) { - shadow_wqe->header.error_code = IB_WC_GENERAL_ERR; - shadow_queue_advance_next_to_complete(&qp->shadow_sq); - } + if (list_empty(&qp->send_err_node)) + list_add_tail(&qp->send_err_node, &cq->send_err_qp_list); spin_unlock_irqrestore(&cq->cq_lock, flags); if (cq->ibcq.comp_handler) cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); +} - mana_put_qp_ref(qp); +void mana_drain_gsi_sqs(struct mana_ib_dev *mdev) +{ + struct mana_ib_qp *qp; + u32 port; + + /* One GSI QP per port, indexed in the QP table by (port << 24 | MANA_GSI_QPN) */ + for (port = 1; port <= mdev->ib_dev.phys_port_cnt; port++) { + qp = mana_get_qp_ref(mdev, (port << 24) | MANA_GSI_QPN, false); + if (!qp) + continue; + + mana_drain_gsi_sq(qp); + mana_put_qp_ref(qp); + } } int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { struct mana_ib_cq *cq = container_of(ibcq, struct mana_ib_cq, ibcq); struct mana_ib_dev *mdev = container_of(ibcq->device, struct mana_ib_dev, ib_dev); + struct mana_cq_poll poll = { .wc = wc, .budget = num_entries, .produced = 0 }; struct gdma_queue *queue = cq->queue.kmem; - struct gdma_comp gdma_cqe; unsigned long flags; - int num_polled = 0; - int comp_read, i; + bool consumed; + int comp_read; + + if (!queue) + return -EINVAL; spin_lock_irqsave(&cq->cq_lock, flags); - for (i = 0; i < num_entries; i++) { - comp_read = mana_gd_poll_cq(queue, &gdma_cqe, 1); - if (comp_read < 1) - break; - mana_handle_cqe(mdev, &gdma_cqe); + while (poll.produced < poll.budget) { + if (!cq->has_pending_cqe) { + comp_read = mana_gd_poll_cq(queue, &cq->pending_cqe, 1); + if (comp_read < 0) { + if (!poll.produced) + poll.produced = comp_read; + goto out; + } + if (!comp_read) + break; + + cq->poll_credit--; + if (!cq->poll_credit) + mana_ib_cq_doorbell(cq, 0); + } + + consumed = mana_handle_cqe(cq, mdev, &poll); + cq->has_pending_cqe = !consumed; } - num_polled = mana_process_completions(cq, num_entries, wc); + mana_flush_completions(cq, &poll); +out: spin_unlock_irqrestore(&cq->cq_lock, flags); - return num_polled; + return poll.produced; } diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c index a8d19586a78a..62c7803ca6c6 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -103,8 +103,7 @@ static int mana_ib_netdev_event(struct notifier_block *this, ib_device_set_netdev(&dev->ib_dev, ndev, i + 1); /* mana_get_primary_netdev() returns ndev with refcount held */ - if (ndev) - netdev_put(ndev, &dev->dev_tracker); + netdev_put(ndev, &dev->dev_tracker); return NOTIFY_OK; default: diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c index e4414d208d4a..858bbc325eb2 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -247,6 +247,9 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext, ucontext->doorbell = doorbell_page; ucmd_resp.comp_mask = MANA_IB_UCNTX_ALLOC_PDN_SUPPORT; + ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_EXT_SUPPORT; + if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT) + ucmd_resp.comp_mask |= MANA_IB_UCNTX_RC_SQ_POW2_SUPPORT; ret = ib_respond_udata(udata, ucmd_resp); if (ret) return ret; @@ -292,7 +295,7 @@ int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_qu } int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, - struct mana_ib_queue *queue) + struct mana_ib_queue *queue, bool is_cq) { struct ib_umem *umem; int err; @@ -301,7 +304,12 @@ int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, queue->id = INVALID_QUEUE_ID; queue->gdma_region = GDMA_INVALID_DMA_REGION; - umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); + if (is_cq) + umem = ib_umem_get_cq_buf_or_va(&mdev->ib_dev, NULL, addr, + size, IB_ACCESS_LOCAL_WRITE); + else + umem = ib_umem_get_va(&mdev->ib_dev, addr, size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem); return PTR_ERR(umem); @@ -592,7 +600,11 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, immutable->gid_tbl_len = attr.gid_tbl_len; if (mana_ib_is_rnic(dev)) { - if (port_num == 1) { + bool port_supports_cm = (port_num == 1 || + (dev->adapter_caps.feature_flags & + MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)); + + if (port_supports_cm) { immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; immutable->max_mad_size = IB_MGMT_MAD_SIZE; } else { @@ -670,10 +682,14 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port, ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width); props->pkey_tbl_len = 1; if (mana_ib_is_rnic(dev)) { + bool port_supports_cm = (port == 1 || + (dev->adapter_caps.feature_flags & + MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)); + props->gid_tbl_len = 16; props->ip_gids = true; props->max_msg_sz = SZ_16M; - if (port == 1) + if (port_supports_cm) props->port_cap_flags = IB_PORT_CM_SUP; } @@ -1059,6 +1075,9 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, req.max_recv_sge = attr->cap.max_recv_sge; req.flags = flags; + if (flags & MANA_RC_FLAG_FIXED_SIZE_WQE) + req.wqe_size_in_bu = qp->rc_qp.wqe_size_in_bu; + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); if (err) return err; @@ -1134,6 +1153,7 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, struct gdma_context *gc = mdev_to_gc(mdev); struct mana_rnic_create_udqp_resp resp = {}; struct mana_rnic_create_udqp_req req = {}; + struct net_device *ndev; int err, i; mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(resp)); @@ -1150,6 +1170,21 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp, req.max_send_sge = attr->cap.max_send_sge; req.max_recv_sge = attr->cap.max_recv_sge; req.qp_type = type; + + /* For GSI QPs, pass the vNIC MAC so the SoC can associate the QP with + * the correct port, transition to INIT, and allocate a per-port QPN. + * MAC must be in reversed byte order. + */ + if (type == IB_QPT_GSI && + (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT)) { + ndev = mana_ib_get_netdev(&mdev->ib_dev, attr->port_num); + if (ndev) { + copy_in_reverse(req.mac, ndev->dev_addr, ETH_ALEN); + req.flags = MANA_UD_QP_FLAG_CREATE_IN_INIT; + req.hdr.req.msg_version = GDMA_MESSAGE_V2; + } + } + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); if (err) return err; diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h index f69866696776..9173ca64d2e1 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -24,8 +24,12 @@ /* MANA doesn't have any limit for MR size */ #define MANA_IB_MAX_MR_SIZE U64_MAX -/* Send queue ID mask */ -#define MANA_SENDQ_MASK BIT(31) +/* + * Send queue ID mask. Queue IDs are 2-bit aligned (see MANA_QID_SUBTYPE_MASK), + * so bit 0 is always free to tag send queues in the lookup table. This keeps + * the whole top byte available to index per-port GSI QPs by (port << 24). + */ +#define MANA_SENDQ_MASK BIT(0) /* Queue ID encodes type in the lower 2 bits */ #define MANA_QID_SUBTYPE_MASK 0x3 @@ -162,17 +166,20 @@ struct mana_ib_cq { struct mana_ib_queue queue; /* protects CQ polling */ spinlock_t cq_lock; - struct list_head list_send_qp; - struct list_head list_recv_qp; + struct list_head send_err_qp_list; + struct list_head recv_err_qp_list; + struct gdma_comp pending_cqe; + bool has_pending_cqe; int cqe; u32 comp_vector; + u32 poll_credit; mana_handle_t cq_handle; }; enum mana_rc_queue_type { MANA_RC_SEND_QUEUE_REQUESTER = 0, MANA_RC_SEND_QUEUE_RESPONDER, - MANA_RC_SEND_QUEUE_FMR, + MANA_RC_SEND_QUEUE_MMQ, MANA_RC_RECV_QUEUE_REQUESTER, MANA_RC_RECV_QUEUE_RESPONDER, MANA_RC_QUEUE_TYPE_MAX, @@ -180,6 +187,7 @@ enum mana_rc_queue_type { struct mana_ib_rc_qp { struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX]; + u32 wqe_size_in_bu; }; enum mana_uc_queue_type { @@ -201,7 +209,6 @@ enum mana_ud_queue_type { struct mana_ib_ud_qp { struct mana_ib_queue queues[MANA_UD_QUEUE_TYPE_MAX]; - u32 sq_psn; }; struct mana_ib_qp { @@ -217,9 +224,14 @@ struct mana_ib_qp { /* The port on the IB device, starting with 1 */ u32 port; + u32 sq_psn; + bool sq_sig_all; - struct list_head cq_send_list; - struct list_head cq_recv_list; + /* Serializes QP modification and error-list transitions. */ + struct mutex modify_lock; + + struct list_head send_err_node; + struct list_head recv_err_node; struct shadow_queue shadow_rq; struct shadow_queue shadow_sq; @@ -262,6 +274,9 @@ enum mana_ib_adapter_features { MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4), MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5), MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6), + MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT = BIT(7), + MANA_IB_FEATURE_RC_QP_SQ_POW2_SUPPORT = BIT(14), + MANA_IB_FEATURE_MULTI_PORT_GSI_SUPPORT = BIT(15), }; struct mana_ib_query_adapter_caps_resp { @@ -372,7 +387,9 @@ struct mana_rnic_destroy_cq_resp { }; /* HW Data */ enum mana_rnic_create_rc_flags { - MANA_RC_FLAG_NO_FMR = 2, + MANA_RC_FLAG_NO_MMQ = BIT(1), + MANA_RC_FLAG_FIXED_SIZE_WQE = BIT(3), + MANA_RC_FLAG_MSN_IN_WQE = BIT(4), }; struct mana_rnic_create_qp_req { @@ -389,7 +406,8 @@ struct mana_rnic_create_qp_req { u32 max_recv_wr; u32 max_send_sge; u32 max_recv_sge; - u32 reserved; + u8 wqe_size_in_bu; + u8 reserved[3]; }; /* HW Data */ struct mana_rnic_create_qp_resp { @@ -445,8 +463,14 @@ struct mana_rnic_create_udqp_req { u32 max_recv_wr; u32 max_send_sge; u32 max_recv_sge; + u8 mac[ETH_ALEN]; /* V2: port MAC for multi-port GSI */ + u16 flags; /* V2: MANA_UD_QP_FLAG_* */ }; /* HW Data */ +enum mana_ud_qp_flags { + MANA_UD_QP_FLAG_CREATE_IN_INIT = BIT(0), +}; + struct mana_rnic_create_udqp_resp { struct gdma_resp_hdr hdr; mana_handle_t qp_handle; @@ -508,7 +532,16 @@ struct mana_rnic_set_qp_state_resp { enum WQE_OPCODE_TYPES { WQE_TYPE_UD_SEND = 0, + WQE_TYPE_RC_SEND = 2, + WQE_TYPE_RC_SEND_IMM = 3, + WQE_TYPE_RC_SEND_INV = 4, + WQE_TYPE_WRITE = 5, + WQE_TYPE_WRITE_IMM = 6, + WQE_TYPE_READ = 7, WQE_TYPE_UD_RECV = 8, + WQE_TYPE_RC_RECV = 9, + WQE_TYPE_REG_MR = 10, + WQE_TYPE_LOCAL_INV = 12, }; /* HW DATA */ struct rdma_send_oob { @@ -527,7 +560,83 @@ struct rdma_send_oob { u32 reserved1; u32 reserved2; } ud_send; + union { + u32 immediate; + u32 invalidate_key; + } rc_send; + struct { + u32 address_hi; + u32 address_low; + u32 rkey; + u32 dma_len; + } rdma; + struct { + u32 mkey; + } mm; }; + union { + u32 immediate_ext; + struct { + u16 rsn; + u16 reserved; + } read; + }; + u32 fsn : 24; + u32 reserved2 : 8; +}; /* HW DATA */ + +struct rdma_recv_oob { + u32 psn_start : 24; + u32 reserved1 : 8; + u32 msn : 24; + u32 reserved2 : 8; +}; /* HW DATA */ + +enum mana_ib_error_code { + VENDOR_ERR_OK = 0x0, + VENDOR_ERR_RX_PKT_LEN = 0x05, + VENDOR_ERR_RX_MSG_LEN_OVFL = 0x102, + VENDOR_ERR_RX_MISBEHAVING_CLIENT = 0x108, + VENDOR_ERR_RX_MALFORMED_WQE = 0x109, + VENDOR_ERR_RX_CLIENT_ID = 0x10a, + VENDOR_ERR_RX_GFID = 0x10b, + VENDOR_ERR_RX_PCIE = 0x10c, + VENDOR_ERR_RX_NO_AVAIL_WQE = 0x111, + VENDOR_ERR_RX_ATB_SGE_MISSCONFIG = 0x143, + VENDOR_ERR_RX_ATB_WQE_MISCONFIG = 0x145, + VENDOR_ERR_RX_ATB_SGE_ADDR_RIGHT = 0x183, + VENDOR_ERR_RX_ATB_WQE_ADDR_RIGHT = 0x185, + VENDOR_ERR_RX_ATB_SGE_ADDR_RANGE = 0x1c3, + VENDOR_ERR_RX_ATB_WQE_ADDR_RANGE = 0x1c5, + VENDOR_ERR_RX_NOT_EMPTY_ON_DISABLE = 0x1c7, + VENDOR_ERR_TX_GDMA_CORRUPTED_WQE = 0x201, + VENDOR_ERR_TX_ATB_WQE_ACCESS_VIOLATION = 0x202, + VENDOR_ERR_TX_ATB_WQE_ADDR_RANGE = 0x203, + VENDOR_ERR_TX_ATB_WQE_CONFIG_ERR = 0x204, + VENDOR_ERR_TX_PCIE_WQE = 0x205, + VENDOR_ERR_TX_ATB_MSG_ACCESS_VIOLATION = 0x206, + VENDOR_ERR_TX_ATB_MSG_ADDR_RANGE = 0x207, + VENDOR_ERR_TX_ATB_MSG_CONFIG_ERR = 0x208, + VENDOR_ERR_TX_PCIE_MSG = 0x209, + VENDOR_ERR_TX_GDMA_INVALID_STATE = 0x20a, + VENDOR_ERR_TX_MISBEHAVING_CLIENT = 0x20b, + VENDOR_ERR_TX_RDMA_MALFORMED_WQE_SIZE = 0x210, + VENDOR_ERR_TX_RDMA_MALFORMED_WQE_FIELD = 0x211, + VENDOR_ERR_TX_RDMA_INVALID_STATE = 0x212, + VENDOR_ERR_TX_RDMA_INVALID_NPT = 0x213, + VENDOR_ERR_TX_RDMA_INVALID_SGID = 0x214, + VENDOR_ERR_TX_RDMA_WQE_UNSUPPORTED = 0x215, + VENDOR_ERR_TX_RDMA_WQE_LEN_ERR = 0x216, + VENDOR_ERR_TX_RDMA_MTU_ERR = 0x217, + VENDOR_ERR_TX_RDMA_VFID_MISMATCH = 0x218, + VENDOR_ERR_HW_MAX = 0x3ff, + /* SW vendor errors */ + VENDOR_ERR_SW_FLUSHED = 0xfff, +}; + +enum mana_ib_cqe_type { + CQE_TYPE_UD_SEND = 1, + CQE_TYPE_UD_SEND_IMM = 2, }; /* HW DATA */ struct mana_rdma_cqe { @@ -538,8 +647,7 @@ struct mana_rdma_cqe { }; struct { u32 cqe_type : 8; - u32 vendor_error : 9; - u32 reserved1 : 15; + u32 reserved1 : 24; u32 sge_offset : 5; u32 tx_wqe_offset : 27; } ud_send; @@ -714,7 +822,7 @@ int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum gdma_queue_type type, struct mana_ib_queue *queue); int mana_ib_create_queue(struct mana_ib_dev *mdev, u64 addr, u32 size, - struct mana_ib_queue *queue); + struct mana_ib_queue *queue, bool is_cq); void mana_ib_destroy_queue(struct mana_ib_dev *mdev, struct mana_ib_queue *queue); struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 8b7be1255c0d..15c5d6cefd62 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -340,7 +340,8 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_dbg(&mdev->ib_dev, "ucmd sq_buf_addr 0x%llx port %u\n", ucmd.sq_buf_addr, ucmd.port); - err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq); + err = mana_ib_create_queue(mdev, ucmd.sq_buf_addr, ucmd.sq_buf_size, &qp->raw_sq, + false); if (err) { ibdev_dbg(&mdev->ib_dev, "Failed to create queue for create qp-raw, err %d\n", err); @@ -422,6 +423,11 @@ static u32 mana_ib_wqe_size(u32 sge, u32 oob_size) return ALIGN(wqe_size, GDMA_WQE_BU_SIZE); } +static u32 mana_ib_fixed_wqe_size(u32 sge, u32 oob_size) +{ + return roundup_pow_of_two(mana_ib_wqe_size(sge, oob_size)); +} + static u32 mana_ib_queue_size(struct ib_qp_init_attr *attr, u32 queue_type) { u32 queue_size; @@ -509,8 +515,20 @@ static int mana_table_store_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) if (err) goto err_remove_sq; + /* GSI QPs are additionally indexed by (port << 24 | MANA_GSI_QPN) so the + * per-port GSI SQ drain can find each of them by iterating ports. + */ + if (qp->ibqp.qp_type == IB_QPT_GSI) { + err = mana_table_store_qp_qid(mdev, qp, + (qp->port << 24) | MANA_GSI_QPN, false); + if (err) + goto err_remove_rq; + } + return 0; +err_remove_rq: + mana_table_remove_qp_qid(mdev, rq->id, false); err_remove_sq: mana_table_remove_qp_qid(mdev, sq->id, true); mana_table_drain_qp_ref(qp); @@ -529,6 +547,8 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp mana_table_remove_qp_qid(mdev, sq->id, true); mana_table_remove_qp_qid(mdev, rq->id, false); + if (qp->ibqp.qp_type == IB_QPT_GSI) + mana_table_remove_qp_qid(mdev, (qp->port << 24) | MANA_GSI_QPN, false); mana_table_drain_qp_ref(qp); } @@ -549,27 +569,42 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext); doorbell = mana_ucontext->doorbell; - flags = MANA_RC_FLAG_NO_FMR; - err = ib_copy_validate_udata_in(udata, ucmd, queue_size); + flags = MANA_RC_FLAG_NO_MMQ; + err = ib_copy_validate_udata_in_cm(udata, ucmd, queue_size, + MANA_IB_RC_QP_FIXED_WQE | MANA_IB_RC_MMQ_CREATE); if (err) return err; for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { - /* skip FMR for user-level RC QPs */ - if (i == MANA_RC_SEND_QUEUE_FMR) { - qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; - qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; + if (i == MANA_RC_SEND_QUEUE_MMQ) { + if (ucmd.comp_mask & MANA_IB_RC_MMQ_CREATE) { + flags &= ~MANA_RC_FLAG_NO_MMQ; + err = mana_ib_create_queue(mdev, ucmd.mmq_buf, ucmd.mmq_size, + &qp->rc_qp.queues[i], false); + if (err) + goto destroy_queues; + } else { + qp->rc_qp.queues[i].id = INVALID_QUEUE_ID; + qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION; + } continue; } err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j], - &qp->rc_qp.queues[i]); - if (err) { - ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err); + &qp->rc_qp.queues[i], false); + if (err) goto destroy_queues; - } j++; } + if (ucmd.comp_mask & MANA_IB_RC_QP_FIXED_WQE) { + u32 wqe_size = mana_ib_fixed_wqe_size(attr->cap.max_send_sge, + INLINE_OOB_EXTRA_LARGE_SIZE); + flags |= MANA_RC_FLAG_FIXED_SIZE_WQE; + if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_MSN_IN_WQE_SUPPORT) + flags |= MANA_RC_FLAG_MSN_IN_WQE; + qp->rc_qp.wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; + } + err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create rc qp %d\n", err); @@ -580,8 +615,10 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, if (udata) { for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) { - if (i == MANA_RC_SEND_QUEUE_FMR) + if (i == MANA_RC_SEND_QUEUE_MMQ) { + resp.mmq_id = qp->rc_qp.queues[i].id; continue; + } resp.queue_id[j] = qp->rc_qp.queues[i].id; j++; } @@ -627,7 +664,7 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, for (i = 0; i < MANA_UC_QUEUE_TYPE_MAX; ++i) { err = mana_ib_create_queue(mdev, ucmd.queue_buf[i], ucmd.queue_size[i], - &qp->uc_qp.queues[i]); + &qp->uc_qp.queues[i], false); if (err) goto destroy_queues; } @@ -658,33 +695,23 @@ static int mana_ib_create_uc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, return err; } -static void mana_add_qp_to_cqs(struct mana_ib_qp *qp) +static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp, bool reset) { struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); unsigned long flags; spin_lock_irqsave(&send_cq->cq_lock, flags); - list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp); + list_del_init(&qp->send_err_node); + /* Keep shadow reset serialized with hardware polling and SW flushing. */ + if (reset) + reset_shadow_queue(&qp->shadow_sq); spin_unlock_irqrestore(&send_cq->cq_lock, flags); spin_lock_irqsave(&recv_cq->cq_lock, flags); - list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp); - spin_unlock_irqrestore(&recv_cq->cq_lock, flags); -} - -static void mana_remove_qp_from_cqs(struct mana_ib_qp *qp) -{ - struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); - struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq); - unsigned long flags; - - spin_lock_irqsave(&send_cq->cq_lock, flags); - list_del(&qp->cq_send_list); - spin_unlock_irqrestore(&send_cq->cq_lock, flags); - - spin_lock_irqsave(&recv_cq->cq_lock, flags); - list_del(&qp->cq_recv_list); + list_del_init(&qp->recv_err_node); + if (reset) + reset_shadow_queue(&qp->shadow_rq); spin_unlock_irqrestore(&recv_cq->cq_lock, flags); } @@ -712,13 +739,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, doorbell = mdev->gdma_dev->doorbell; err = create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr, - sizeof(struct ud_rq_shadow_wqe)); + sizeof(struct shadow_wqe_header)); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create shadow rq err %d\n", err); goto destroy_queues; } err = create_shadow_queue(&qp->shadow_sq, attr->cap.max_send_wr, - sizeof(struct ud_sq_shadow_wqe)); + sizeof(struct shadow_wqe_header)); if (err) { ibdev_err(&mdev->ib_dev, "Failed to create shadow sq err %d\n", err); goto destroy_shadow_queues; @@ -729,7 +756,8 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, ibdev_err(&mdev->ib_dev, "Failed to create ud qp %d\n", err); goto destroy_shadow_queues; } - qp->ibqp.qp_num = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; + qp->ibqp.qp_num = (qp->ibqp.qp_type == IB_QPT_GSI) ? + MANA_GSI_QPN : qp->ud_qp.queues[MANA_UD_RECV_QUEUE].id; qp->port = attr->port_num; for (i = 0; i < MANA_UD_QUEUE_TYPE_MAX; ++i) @@ -739,8 +767,6 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, if (err) goto destroy_qp; - mana_add_qp_to_cqs(qp); - return 0; destroy_qp: @@ -757,6 +783,13 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr, struct ib_udata *udata) { + struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); + + qp->sq_sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR; + mutex_init(&qp->modify_lock); + INIT_LIST_HEAD(&qp->send_err_node); + INIT_LIST_HEAD(&qp->recv_err_node); + switch (attr->qp_type) { case IB_QPT_RAW_PACKET: /* When rwq_ind_tbl is used, it's for creating WQs for RSS */ @@ -854,19 +887,56 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, return 0; } +static void mana_ib_modify_qp_state(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, struct ib_udata *udata) +{ + struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); + + if (udata) + return; + + if (attr_mask & IB_QP_STATE) { + switch (attr->qp_state) { + case IB_QPS_RESET: + mana_remove_qp_from_cqs(qp, true); + break; + default: + break; + } + } + + if (attr_mask & IB_QP_SQ_PSN) + qp->sq_psn = attr->sq_psn; +} + int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { + struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); + int ret; + + mutex_lock(&qp->modify_lock); + switch (ibqp->qp_type) { case IB_QPT_RC: case IB_QPT_UC: case IB_QPT_UD: case IB_QPT_GSI: - return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata); + ret = mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata); + if (ret) + goto out_unlock; + break; default: - ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; + goto out_unlock; } + + mana_ib_modify_qp_state(ibqp, attr, attr_mask, udata); + +out_unlock: + mutex_unlock(&qp->modify_lock); + + return ret; } static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, @@ -1003,8 +1073,8 @@ static int mana_ib_destroy_ud_qp(struct mana_ib_qp *qp, struct ib_udata *udata) if (err) return err; - mana_remove_qp_from_cqs(qp); mana_table_remove_qp(mdev, qp); + mana_remove_qp_from_cqs(qp, false); destroy_shadow_queue(&qp->shadow_rq); destroy_shadow_queue(&qp->shadow_sq); diff --git a/drivers/infiniband/hw/mana/shadow_queue.h b/drivers/infiniband/hw/mana/shadow_queue.h index a4b3818f9c39..b5d6bb28bd45 100644 --- a/drivers/infiniband/hw/mana/shadow_queue.h +++ b/drivers/infiniband/hw/mana/shadow_queue.h @@ -6,21 +6,19 @@ #ifndef _MANA_SHADOW_QUEUE_H_ #define _MANA_SHADOW_QUEUE_H_ +#include + +#define MANA_WQ_FENCE_WC BIT(0) +#define MANA_WQ_NO_SIGNAL_WC BIT(1) +#define MANA_WQE_OFFSET_MASK GENMASK(23, 0) + struct shadow_wqe_header { - u16 opcode; - u16 error_code; - u32 posted_wqe_size; u64 wr_id; -}; - -struct ud_rq_shadow_wqe { - struct shadow_wqe_header header; - u32 byte_len; - u32 src_qpn; -}; - -struct ud_sq_shadow_wqe { - struct shadow_wqe_header header; + u64 wqe_offset_or_psn : 24; + u64 wqe_size_in_bu : 8; + u64 fsn : 24; + u64 send_opcode : 4; + u64 flags : 2; }; struct shadow_queue { @@ -28,8 +26,6 @@ struct shadow_queue { u64 prod_idx; /* Unmasked consumer index, Incremented on cq polling */ u64 cons_idx; - /* Unmasked index of next-to-complete (from HW) shadow WQE */ - u64 next_to_complete_idx; /* queue size in wqes */ u32 length; /* distance between elements in bytes */ @@ -50,6 +46,12 @@ static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t lengt return 0; } +static inline void reset_shadow_queue(struct shadow_queue *queue) +{ + queue->prod_idx = 0; + queue->cons_idx = 0; +} + static inline void destroy_shadow_queue(struct shadow_queue *queue) { kvfree(queue->buffer); @@ -57,12 +59,14 @@ static inline void destroy_shadow_queue(struct shadow_queue *queue) static inline bool shadow_queue_full(struct shadow_queue *queue) { - return (queue->prod_idx - queue->cons_idx) >= queue->length; + /* Do not reuse an entry until the poller has finished reading it. */ + return (queue->prod_idx - smp_load_acquire(&queue->cons_idx)) >= queue->length; } static inline bool shadow_queue_empty(struct shadow_queue *queue) { - return queue->prod_idx == queue->cons_idx; + /* Pair with posting's release of the initialized shadow WQE. */ + return smp_load_acquire(&queue->prod_idx) == queue->cons_idx; } static inline void * @@ -82,34 +86,23 @@ shadow_queue_producer_entry(struct shadow_queue *queue) static inline void * shadow_queue_get_next_to_consume(const struct shadow_queue *queue) { - if (queue->cons_idx == queue->next_to_complete_idx) + /* The producer publishes the WQE before advancing prod_idx. */ + if (queue->cons_idx == smp_load_acquire(&queue->prod_idx)) return NULL; return shadow_queue_get_element(queue, queue->cons_idx); } -static inline void * -shadow_queue_get_next_to_complete(struct shadow_queue *queue) -{ - if (queue->next_to_complete_idx == queue->prod_idx) - return NULL; - - return shadow_queue_get_element(queue, queue->next_to_complete_idx); -} - static inline void shadow_queue_advance_producer(struct shadow_queue *queue) { - queue->prod_idx++; + /* Publish all WQE fields to CQ polling on another CPU. */ + smp_store_release(&queue->prod_idx, queue->prod_idx + 1); } static inline void shadow_queue_advance_consumer(struct shadow_queue *queue) { - queue->cons_idx++; -} - -static inline void shadow_queue_advance_next_to_complete(struct shadow_queue *queue) -{ - queue->next_to_complete_idx++; + /* Finish WC generation and queue-tail updates before allowing reuse. */ + smp_store_release(&queue->cons_idx, queue->cons_idx + 1); } #endif diff --git a/drivers/infiniband/hw/mana/wq.c b/drivers/infiniband/hw/mana/wq.c index 6b066d605dcb..4042f26632c0 100644 --- a/drivers/infiniband/hw/mana/wq.c +++ b/drivers/infiniband/hw/mana/wq.c @@ -25,7 +25,8 @@ struct ib_wq *mana_ib_create_wq(struct ib_pd *pd, ibdev_dbg(&mdev->ib_dev, "ucmd wq_buf_addr 0x%llx\n", ucmd.wq_buf_addr); - err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue); + err = mana_ib_create_queue(mdev, ucmd.wq_buf_addr, ucmd.wq_buf_size, &wq->queue, + false); if (err) { ibdev_dbg(&mdev->ib_dev, "Failed to create queue for create wq, %d\n", err); diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c index 36a1d506f08f..250d673f58f2 100644 --- a/drivers/infiniband/hw/mana/wr.c +++ b/drivers/infiniband/hw/mana/wr.c @@ -7,29 +7,30 @@ #define MAX_WR_SGL_NUM (2) -static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr) +static int mana_ib_post_rq(struct mana_ib_qp *qp, const struct ib_recv_wr *wr) { - struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); - struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_RECV_QUEUE].kmem; + struct ib_sge mana_ib_dummy_sge = {.addr = 1, .length = 0, .lkey = 0}; + struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp); struct gdma_posted_wqe_info wqe_info = {0}; - struct gdma_sge gdma_sgl[MAX_WR_SGL_NUM]; + struct gdma_queue *queue = ib_rq->kmem; struct gdma_wqe_request wqe_req = {0}; - struct ud_rq_shadow_wqe *shadow_wqe; - int err, i; + struct shadow_wqe_header *shadow_wqe; + int err; if (shadow_queue_full(&qp->shadow_rq)) return -EINVAL; - if (wr->num_sge > MAX_WR_SGL_NUM) + if (wr->num_sge > MAX_RX_WQE_SGL_ENTRIES) return -EINVAL; - for (i = 0; i < wr->num_sge; ++i) { - gdma_sgl[i].address = wr->sg_list[i].addr; - gdma_sgl[i].mem_key = wr->sg_list[i].lkey; - gdma_sgl[i].size = wr->sg_list[i].length; - } wqe_req.num_sge = wr->num_sge; - wqe_req.sgl = gdma_sgl; + wqe_req.ib_sgl = wr->sg_list; + wqe_req.flags = GDMA_WR_IB_SGL; + + if (wr->num_sge == 0) { + wqe_req.ib_sgl = &mana_ib_dummy_sge; + wqe_req.num_sge = 1; + } err = mana_gd_post_work_request(queue, &wqe_req, &wqe_info); if (err) @@ -37,42 +38,55 @@ static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr * shadow_wqe = shadow_queue_producer_entry(&qp->shadow_rq); memset(shadow_wqe, 0, sizeof(*shadow_wqe)); - shadow_wqe->header.opcode = IB_WC_RECV; - shadow_wqe->header.wr_id = wr->wr_id; - shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu; + shadow_wqe->wr_id = wr->wr_id; + shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; + shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset; shadow_queue_advance_producer(&qp->shadow_rq); - mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); return 0; } +static int mana_ib_post_recv_ud(struct mana_ib_qp *qp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); + struct mana_ib_queue *ib_rq = mana_qp_get_rq(qp); + struct gdma_queue *rq = ib_rq->kmem; + bool ring_rq = false; + int err = 0; + + for (; wr; wr = wr->next) { + err = mana_ib_post_rq(qp, wr); + if (unlikely(err)) { + *bad_wr = wr; + break; + } + ring_rq = true; + } + + if (ring_rq) + mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), rq); + + return err; +} + int mana_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); - int err = 0; - for (; wr; wr = wr->next) { - switch (ibqp->qp_type) { - case IB_QPT_UD: - case IB_QPT_GSI: - err = mana_ib_post_recv_ud(qp, wr); - if (unlikely(err)) { - *bad_wr = wr; - return err; - } - break; - default: - ibdev_dbg(ibqp->device, "Posting recv wr on qp type %u is not supported\n", - ibqp->qp_type); - return -EINVAL; - } + switch (ibqp->qp_type) { + case IB_QPT_UD: + case IB_QPT_GSI: + return mana_ib_post_recv_ud(qp, wr, bad_wr); + default: + /* Unsupported QP type */ + *bad_wr = wr; + return -EINVAL; } - - return err; } -static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr) +static int mana_ib_post_send_ud_one(struct mana_ib_qp *qp, const struct ib_ud_wr *wr) { struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); struct mana_ib_ah *ah = container_of(wr->ah, struct mana_ib_ah, ibah); @@ -82,14 +96,12 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr struct gdma_posted_wqe_info wqe_info = {0}; struct gdma_wqe_request wqe_req = {0}; struct rdma_send_oob send_oob = {0}; - struct ud_sq_shadow_wqe *shadow_wqe; + struct shadow_wqe_header *shadow_wqe; + bool signaled = qp->sq_sig_all || (wr->wr.send_flags & IB_SEND_SIGNALED); int err, i; - if (!ndev) { - ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n", - qp->port, qp->ibqp.qp_num); + if (!ndev) return -EINVAL; - } if (wr->wr.opcode != IB_WR_SEND) return -EINVAL; @@ -111,16 +123,16 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr wqe_req.num_sge = wr->wr.num_sge + 1; wqe_req.sgl = gdma_sgl; - wqe_req.inline_oob_size = sizeof(struct rdma_send_oob); + wqe_req.inline_oob_size = INLINE_OOB_LARGE_SIZE; wqe_req.inline_oob_data = &send_oob; wqe_req.flags = GDMA_WR_OOB_IN_SGL; wqe_req.client_data_unit = ib_mtu_enum_to_int(ib_mtu_int_to_enum(ndev->mtu)); send_oob.wqe_type = WQE_TYPE_UD_SEND; send_oob.fence = !!(wr->wr.send_flags & IB_SEND_FENCE); - send_oob.signaled = !!(wr->wr.send_flags & IB_SEND_SIGNALED); + send_oob.signaled = signaled; send_oob.solicited = !!(wr->wr.send_flags & IB_SEND_SOLICITED); - send_oob.psn = qp->ud_qp.sq_psn; + send_oob.psn = qp->sq_psn; send_oob.ssn_or_rqpn = wr->remote_qpn; send_oob.ud_send.remote_qkey = qp->ibqp.qp_type == IB_QPT_GSI ? IB_QP1_QKEY : wr->remote_qkey; @@ -129,40 +141,64 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr if (err) return err; - qp->ud_qp.sq_psn++; + qp->sq_psn++; shadow_wqe = shadow_queue_producer_entry(&qp->shadow_sq); memset(shadow_wqe, 0, sizeof(*shadow_wqe)); - shadow_wqe->header.opcode = IB_WC_SEND; - shadow_wqe->header.wr_id = wr->wr.wr_id; - shadow_wqe->header.posted_wqe_size = wqe_info.wqe_size_in_bu; + shadow_wqe->wr_id = wr->wr.wr_id; + shadow_wqe->flags = signaled ? 0 : MANA_WQ_NO_SIGNAL_WC; + shadow_wqe->wqe_size_in_bu = wqe_info.wqe_size_in_bu; + shadow_wqe->wqe_offset_or_psn = wqe_info.wqe_offset; shadow_queue_advance_producer(&qp->shadow_sq); - mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); return 0; } +static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct mana_ib_dev *mdev = container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); + struct mana_ib_cq *cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq); + struct gdma_queue *queue = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].kmem; + unsigned long flags; + bool ring_sq = false; + bool flush_send; + int err = 0; + + for (; wr; wr = wr->next) { + err = mana_ib_post_send_ud_one(qp, ud_wr(wr)); + if (unlikely(err)) { + *bad_wr = wr; + break; + } + ring_sq = true; + } + + if (ring_sq) { + mana_gd_wq_ring_doorbell(mdev_to_gc(mdev), queue); + + spin_lock_irqsave(&cq->cq_lock, flags); + flush_send = !list_empty(&qp->send_err_node); + spin_unlock_irqrestore(&cq->cq_lock, flags); + + if (flush_send && cq->ibcq.comp_handler) + cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); + } + + return err; +} + int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { - int err = 0; struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); - for (; wr; wr = wr->next) { - switch (ibqp->qp_type) { - case IB_QPT_UD: - case IB_QPT_GSI: - err = mana_ib_post_send_ud(qp, ud_wr(wr)); - if (unlikely(err)) { - *bad_wr = wr; - return err; - } - break; - default: - ibdev_dbg(ibqp->device, "Posting send wr on qp type %u is not supported\n", - ibqp->qp_type); - return -EINVAL; - } + switch (ibqp->qp_type) { + case IB_QPT_UD: + case IB_QPT_GSI: + return mana_ib_post_send_ud(qp, wr, bad_wr); + default: + /* Unsupported QP type */ + *bad_wr = wr; + return -EINVAL; } - - return err; } diff --git a/drivers/infiniband/hw/mlx4/cm.c b/drivers/infiniband/hw/mlx4/cm.c index 1e5e525fea73..810f95e55fda 100644 --- a/drivers/infiniband/hw/mlx4/cm.c +++ b/drivers/infiniband/hw/mlx4/cm.c @@ -301,10 +301,20 @@ static void schedule_delayed(struct ib_device *ibdev, struct id_map_entry *id, spin_lock_irqsave(&sriov->going_down_lock, flags); /*make sure that there is no schedule inside the scheduled work.*/ if (!sriov->is_going_down || id->scheduled_delete) { + /* + * Do not requeue timeout work that is already executing. The + * timeout handler may be waiting for id_map_lock and will free + * this entry after the caller releases it. + */ + if (id->scheduled_delete && + !cancel_delayed_work(&id->timeout)) + goto out; + id->scheduled_delete = 1; id->rtu_timeout = rtu_timeout; mod_delayed_work(cm_wq, &id->timeout, timeout); } +out: spin_unlock_irqrestore(&sriov->going_down_lock, flags); } diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 887912469742..81dc47e57352 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -185,9 +185,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq, goto err_umem; } } else { - cq->umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); + cq->umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, + ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->umem)) { err = PTR_ERR(cq->umem); goto err_cq; @@ -354,9 +355,10 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq if (!cq->resize_buf) return -ENOMEM; - cq->resize_umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, - entries * cqe_size, - IB_ACCESS_LOCAL_WRITE); + cq->resize_umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, + ucmd.buf_addr, + entries * cqe_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(cq->resize_umem)) { err = PTR_ERR(cq->resize_umem); goto err_buf; @@ -989,7 +991,8 @@ void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq) * Now sweep backwards through the CQ, removing CQ entries * that match our QP by copying older entries on top of them. */ - while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { + while (prod_index != cq->mcq.cons_index) { + --prod_index; cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); cqe += cqe_inc; diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 7266a6141944..1594e6407b6c 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1169,6 +1169,10 @@ static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) struct mlx4_clock_params params; int ret; + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) + return -EPERM; + vm_flags_clear(vma, VM_MAYWRITE); + ret = mlx4_get_internal_clock_params(dev->dev, ¶ms); if (ret) return ret; diff --git a/drivers/infiniband/hw/mlx4/mcg.c b/drivers/infiniband/hw/mlx4/mcg.c index dc7c0daaaba5..bba178ffe1b2 100644 --- a/drivers/infiniband/hw/mlx4/mcg.c +++ b/drivers/infiniband/hw/mlx4/mcg.c @@ -433,7 +433,7 @@ static u16 cmp_rec(struct ib_sa_mcmember_data *src, } /* release group, return 1 if this was last release and group is destroyed - * timout work is canceled sync */ + * timeout work is canceled sync */ static int release_group(struct mcast_group *group, int from_timeout_handler) { struct mlx4_ib_demux_ctx *ctx = group->demux; @@ -694,7 +694,7 @@ static void mlx4_ib_mcg_work_handler(struct work_struct *work) cur_join_state = group->rec.scope_join_state & 0xf; if (method == IB_MGMT_METHOD_GET_RESP) { - /* successfull join */ + /* successful join */ if (!cur_join_state && resp_join_state) --rc; } else if (!resp_join_state) diff --git a/drivers/infiniband/hw/mlx5/Makefile b/drivers/infiniband/hw/mlx5/Makefile index dd7bb377f491..35afc7bc57d2 100644 --- a/drivers/infiniband/hw/mlx5/Makefile +++ b/drivers/infiniband/hw/mlx5/Makefile @@ -6,7 +6,6 @@ mlx5_ib-y := ah.o \ cong.o \ counters.o \ cq.o \ - data_direct.o \ dm.o \ dmah.o \ doorbell.o \ diff --git a/drivers/infiniband/hw/mlx5/cmd.c b/drivers/infiniband/hw/mlx5/cmd.c index 7c08e3008927..895b62cc528d 100644 --- a/drivers/infiniband/hw/mlx5/cmd.c +++ b/drivers/infiniband/hw/mlx5/cmd.c @@ -245,24 +245,3 @@ int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid) MLX5_SET(dealloc_uar_in, in, uid, uid); return mlx5_cmd_exec_in(dev, dealloc_uar, in); } - -int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct, - char *out_vuid) -{ - u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + - MLX5_ST_SZ_BYTES(array1024_auto)] = {}; - u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {}; - char *vuid; - int err; - - MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID); - MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id)); - MLX5_SET(query_vuid_in, in, data_direct, data_direct); - err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); - if (err) - return err; - - vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid); - memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto)); - return 0; -} diff --git a/drivers/infiniband/hw/mlx5/cmd.h b/drivers/infiniband/hw/mlx5/cmd.h index e6c88b6ebd0d..e5cd31270443 100644 --- a/drivers/infiniband/hw/mlx5/cmd.h +++ b/drivers/infiniband/hw/mlx5/cmd.h @@ -58,6 +58,4 @@ int mlx5_cmd_mad_ifc(struct mlx5_ib_dev *dev, const void *inb, void *outb, u16 opmod, u8 port); int mlx5_cmd_uar_alloc(struct mlx5_core_dev *dev, u32 *uarn, u16 uid); int mlx5_cmd_uar_dealloc(struct mlx5_core_dev *dev, u32 uarn, u16 uid); -int mlx5_cmd_query_vuid(struct mlx5_core_dev *dev, bool data_direct, - char *out_vuid); #endif /* MLX5_IB_CMD_H */ diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 49b4bf148a4a..55ffd7d4de51 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -1117,6 +1117,9 @@ int mlx5_ib_pre_destroy_cq(struct ib_cq *cq) struct mlx5_ib_dev *dev = to_mdev(cq->device); struct mlx5_ib_cq *mcq = to_mcq(cq); + if (!cq->uobject) + cancel_work_sync(&mcq->notify_work); + return mlx5_core_destroy_cq(dev->mdev, &mcq->mcq); } @@ -1169,7 +1172,8 @@ void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 rsn, struct mlx5_ib_srq *srq) /* Now sweep backwards through the CQ, removing CQ entries * that match our QP by copying older entries on top of them. */ - while ((int) --prod_index - (int) cq->mcq.cons_index >= 0) { + while (prod_index != cq->mcq.cons_index) { + --prod_index; cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); cqe64 = (cq->mcq.cqe_sz == 64) ? cqe : cqe + 64; if (is_equal_rsn(cqe64, rsn)) { @@ -1245,9 +1249,9 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, if (ucmd.cqe_size && SIZE_MAX / ucmd.cqe_size <= entries - 1) return -EINVAL; - umem = ib_umem_get_va(&dev->ib_dev, ucmd.buf_addr, - (size_t)ucmd.cqe_size * entries, - IB_ACCESS_LOCAL_WRITE); + umem = ib_umem_get_cq_buf_or_va(&dev->ib_dev, NULL, ucmd.buf_addr, + (size_t)ucmd.cqe_size * entries, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(umem)) { err = PTR_ERR(umem); return err; @@ -1312,7 +1316,7 @@ static int copy_resize_cqes(struct mlx5_ib_cq *cq) while (get_cqe_opcode(scqe64) != MLX5_CQE_RESIZE_CQ) { dcqe = mlx5_frag_buf_get_wqe(&cq->resize_buf->fbc, - (i + 1) & cq->resize_buf->nent); + (i + 1) & (cq->resize_buf->nent - 1)); dcqe64 = dsize == 64 ? dcqe : dcqe + 64; sw_own = sw_ownership_bit(i + 1, cq->resize_buf->nent); memcpy(dcqe, scqe, dsize); diff --git a/drivers/infiniband/hw/mlx5/data_direct.c b/drivers/infiniband/hw/mlx5/data_direct.c deleted file mode 100644 index d57484245c38..000000000000 --- a/drivers/infiniband/hw/mlx5/data_direct.c +++ /dev/null @@ -1,227 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB -/* - * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved - */ - -#include "mlx5_ib.h" -#include "data_direct.h" - -static LIST_HEAD(mlx5_data_direct_dev_list); -static LIST_HEAD(mlx5_data_direct_reg_list); - -/* - * This mutex should be held when accessing either of the above lists - */ -static DEFINE_MUTEX(mlx5_data_direct_mutex); - -struct mlx5_data_direct_registration { - struct mlx5_ib_dev *ibdev; - char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; - struct list_head list; -}; - -static const struct pci_device_id mlx5_data_direct_pci_table[] = { - { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */ - { 0, } -}; - -static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev) -{ - struct pci_dev *pdev = dev->pdev; - unsigned int vpd_size, kw_len; - u8 *vpd_data; - int start; - int ret; - - vpd_data = pci_vpd_alloc(pdev, &vpd_size); - if (IS_ERR(vpd_data)) { - pci_err(pdev, "Unable to read VPD, err=%pe\n", vpd_data); - return PTR_ERR(vpd_data); - } - - start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "VU", &kw_len); - if (start < 0) { - ret = start; - pci_err(pdev, "VU keyword not found, err=%d\n", ret); - goto end; - } - - dev->vuid = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL); - ret = dev->vuid ? 0 : -ENOMEM; - -end: - kfree(vpd_data); - return ret; -} - -static void mlx5_data_direct_shutdown(struct pci_dev *pdev) -{ - pci_disable_device(pdev); -} - -static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) -{ - int err; - - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - if (err) { - dev_warn(&pdev->dev, - "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err); - err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err); - return err; - } - } - - dma_set_max_seg_size(&pdev->dev, SZ_2G); - return 0; -} - -int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid) -{ - struct mlx5_data_direct_registration *reg; - struct mlx5_data_direct_dev *dev; - - reg = kzalloc_obj(*reg); - if (!reg) - return -ENOMEM; - - reg->ibdev = ibdev; - strscpy(reg->vuid, vuid); - - mutex_lock(&mlx5_data_direct_mutex); - list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { - if (strcmp(dev->vuid, vuid) == 0) { - mlx5_ib_data_direct_bind(ibdev, dev); - break; - } - } - - /* Add the registration to its global list, to be used upon bind/unbind - * of its affiliated data direct device - */ - list_add_tail(®->list, &mlx5_data_direct_reg_list); - mutex_unlock(&mlx5_data_direct_mutex); - return 0; -} - -void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev) -{ - struct mlx5_data_direct_registration *reg; - - mutex_lock(&mlx5_data_direct_mutex); - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { - if (reg->ibdev == ibdev) { - list_del(®->list); - kfree(reg); - goto end; - } - } - - WARN_ON(true); -end: - mutex_unlock(&mlx5_data_direct_mutex); -} - -static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) -{ - struct mlx5_data_direct_registration *reg; - - mutex_lock(&mlx5_data_direct_mutex); - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { - if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_ib_data_direct_bind(reg->ibdev, dev); - } - - /* Add the data direct device to the global list, further IB devices may - * use it later as well - */ - list_add_tail(&dev->list, &mlx5_data_direct_dev_list); - mutex_unlock(&mlx5_data_direct_mutex); -} - -static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev) -{ - struct mlx5_data_direct_registration *reg; - - mutex_lock(&mlx5_data_direct_mutex); - /* Prevent any further affiliations */ - list_del(&dev->list); - list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { - if (strcmp(dev->vuid, reg->vuid) == 0) - mlx5_ib_data_direct_unbind(reg->ibdev); - } - mutex_unlock(&mlx5_data_direct_mutex); -} - -static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id) -{ - struct mlx5_data_direct_dev *dev; - int err; - - dev = kzalloc_obj(*dev); - if (!dev) - return -ENOMEM; - - dev->device = &pdev->dev; - dev->pdev = pdev; - - pci_set_drvdata(dev->pdev, dev); - err = pci_enable_device(pdev); - if (err) { - dev_err(dev->device, "Cannot enable PCI device, err=%d\n", err); - goto err; - } - - pci_set_master(pdev); - err = mlx5_data_direct_set_dma_caps(pdev); - if (err) - goto err_disable; - - if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && - pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) && - pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) - dev_dbg(dev->device, "Enabling pci atomics failed\n"); - - err = mlx5_data_direct_vpd_get_vuid(dev); - if (err) - goto err_disable; - - mlx5_data_direct_dev_reg(dev); - return 0; - -err_disable: - pci_disable_device(pdev); -err: - kfree(dev); - return err; -} - -static void mlx5_data_direct_remove(struct pci_dev *pdev) -{ - struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev); - - mlx5_data_direct_dev_unreg(dev); - pci_disable_device(pdev); - kfree(dev->vuid); - kfree(dev); -} - -static struct pci_driver mlx5_data_direct_driver = { - .name = KBUILD_MODNAME, - .id_table = mlx5_data_direct_pci_table, - .probe = mlx5_data_direct_probe, - .remove = mlx5_data_direct_remove, - .shutdown = mlx5_data_direct_shutdown, -}; - -int mlx5_data_direct_driver_register(void) -{ - return pci_register_driver(&mlx5_data_direct_driver); -} - -void mlx5_data_direct_driver_unregister(void) -{ - pci_unregister_driver(&mlx5_data_direct_driver); -} diff --git a/drivers/infiniband/hw/mlx5/data_direct.h b/drivers/infiniband/hw/mlx5/data_direct.h deleted file mode 100644 index 2fd2bdbe8f69..000000000000 --- a/drivers/infiniband/hw/mlx5/data_direct.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ -/* - * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved - */ - -#ifndef _MLX5_IB_DATA_DIRECT_H -#define _MLX5_IB_DATA_DIRECT_H - -struct mlx5_ib_dev; - -struct mlx5_data_direct_dev { - struct device *device; - struct pci_dev *pdev; - char *vuid; - struct list_head list; -}; - -int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid); -void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev); -int mlx5_data_direct_driver_register(void); -void mlx5_data_direct_driver_unregister(void); - -#endif diff --git a/drivers/infiniband/hw/mlx5/fs.c b/drivers/infiniband/hw/mlx5/fs.c index b155baee0017..bc2c15c7172b 100644 --- a/drivers/infiniband/hw/mlx5/fs.c +++ b/drivers/infiniband/hw/mlx5/fs.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "mlx5_ib.h" #include "counters.h" @@ -1647,36 +1648,32 @@ static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *de struct ib_flow_attr *flow_attr, struct mlx5_flow_destination *dst) { + DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_wc, flows, 1); + DEFINE_RAW_FLEX(struct ib_flow_attr, flow_attr_uc, flows, 1); + struct mlx5_ib_flow_handler *handler_ucast = NULL; struct mlx5_ib_flow_handler *handler = NULL; - static struct { - struct ib_flow_spec_eth eth_flow; - struct ib_flow_attr flow_attr; - } leftovers_wc = { .flow_attr = { .num_of_specs = 1, - .size = sizeof(leftovers_wc) }, - .eth_flow = { - .type = IB_FLOW_SPEC_ETH, - .size = sizeof(struct ib_flow_spec_eth), - .mask = { .dst_mac = { 0x1 } }, - .val = { .dst_mac = { 0x1 } } } }; + flow_attr_wc->num_of_specs = 1; + flow_attr_wc->size = sizeof(struct ib_flow_attr) + + sizeof(struct ib_flow_spec_eth); + flow_attr_wc->flows[0].eth.type = IB_FLOW_SPEC_ETH; + flow_attr_wc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth); + flow_attr_wc->flows[0].eth.mask.dst_mac[0] = 0x1; + flow_attr_wc->flows[0].eth.val.dst_mac[0] = 0x1; - static struct { - struct ib_flow_spec_eth eth_flow; - struct ib_flow_attr flow_attr; - } leftovers_uc = { .flow_attr = { .num_of_specs = 1, - .size = sizeof(leftovers_uc) }, - .eth_flow = { - .type = IB_FLOW_SPEC_ETH, - .size = sizeof(struct ib_flow_spec_eth), - .mask = { .dst_mac = { 0x1 } }, - .val = { .dst_mac = {} } } }; + flow_attr_uc->num_of_specs = 1; + flow_attr_uc->size = sizeof(struct ib_flow_attr) + + sizeof(struct ib_flow_spec_eth); + flow_attr_uc->flows[0].eth.type = IB_FLOW_SPEC_ETH; + flow_attr_uc->flows[0].eth.size = sizeof(struct ib_flow_spec_eth); + flow_attr_uc->flows[0].eth.mask.dst_mac[0] = 0x1; - handler = create_flow_rule(dev, ft_prio, &leftovers_wc.flow_attr, dst); + handler = create_flow_rule(dev, ft_prio, flow_attr_wc, dst); if (!IS_ERR(handler) && flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) { handler_ucast = create_flow_rule(dev, ft_prio, - &leftovers_uc.flow_attr, dst); + flow_attr_uc, dst); if (IS_ERR(handler_ucast)) { mlx5_del_flow_rules(handler->rule); ft_prio->refcount--; @@ -3017,6 +3014,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( if (err) return err; + if (priority > MLX5_IB_FLOW_LAST_PRIO) + return -EINVAL; + obj = kzalloc_obj(*obj); if (!obj) return -ENOMEM; @@ -3056,8 +3056,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)( return 0; destroy_res: - --ft_prio->anchor.rule_goto_table_ref; - mlx5_steering_anchor_destroy_res(ft_prio); + if (!--ft_prio->anchor.rule_goto_table_ref) + mlx5_steering_anchor_destroy_res(ft_prio); put_flow_table: put_flow_table(dev, ft_prio, true); free_obj: diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index a457647edacd..65455aedd559 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -51,7 +52,6 @@ #include #include #include "macsec.h" -#include "data_direct.h" #include "dmah.h" #define UVERBS_MODULE_NAME mlx5_ib @@ -3370,7 +3370,7 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev) { struct mlx5_ib_resources *devr = &dev->devr; struct ib_srq_init_attr attr; - struct ib_srq *s0, *s1; + struct ib_srq *s0 = NULL, *s1; int ret = 0; /* @@ -3388,19 +3388,27 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev) if (ret) goto unlock; - memset(&attr, 0, sizeof(attr)); - attr.attr.max_sge = 1; - attr.attr.max_wr = 1; - attr.srq_type = IB_SRQT_XRC; - attr.ext.cq = devr->c0; + /* + * s0 is an XRC-type placeholder SRQ used as the default XRQN for + * XRC QPs. Skip it when XRC is absent; all devr->s0 accesses in + * qp.c are inside XRC QP paths that the verbs layer blocks before + * reaching this driver when xrc=0, so NULL is safe. + */ + if (MLX5_CAP_GEN(dev->mdev, xrc)) { + memset(&attr, 0, sizeof(attr)); + attr.attr.max_sge = 1; + attr.attr.max_wr = 1; + attr.srq_type = IB_SRQT_XRC; + attr.ext.cq = devr->c0; - s0 = ib_create_srq(devr->p0, &attr); - if (IS_ERR(s0)) { - ret = PTR_ERR(s0); - mlx5_ib_err(dev, - "Couldn't create SRQ 0 for res init, err=%pe\n", - s0); - goto unlock; + s0 = ib_create_srq(devr->p0, &attr); + if (IS_ERR(s0)) { + ret = PTR_ERR(s0); + mlx5_ib_err(dev, + "Couldn't create SRQ 0 for res init, err=%pe\n", + s0); + goto unlock; + } } memset(&attr, 0, sizeof(attr)); @@ -3414,7 +3422,8 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev) mlx5_ib_err(dev, "Couldn't create SRQ 1 for res init, err=%pe\n", s1); - ib_destroy_srq(s0); + if (s0) + ib_destroy_srq(s0); goto unlock; } @@ -3431,8 +3440,11 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) struct mlx5_ib_resources *devr = &dev->devr; int ret; + mutex_init(&devr->cq_lock); + mutex_init(&devr->srq_lock); + if (!MLX5_CAP_GEN(dev->mdev, xrc)) - return -EOPNOTSUPP; + return 0; ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0); if (ret) @@ -3444,9 +3456,6 @@ static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev) return ret; } - mutex_init(&devr->cq_lock); - mutex_init(&devr->srq_lock); - return 0; } @@ -3457,10 +3466,13 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev) /* After s0/s1 init, they are not unset during the device lifetime. */ if (devr->s1) { ib_destroy_srq(devr->s1); - ib_destroy_srq(devr->s0); + if (devr->s0) + ib_destroy_srq(devr->s0); + } + if (MLX5_CAP_GEN(dev->mdev, xrc)) { + mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0); + mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0); } - mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0); - mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0); /* After p0/c0 init, they are not unset during the device lifetime. */ if (devr->c0) { ib_destroy_cq(devr->c0); @@ -3470,87 +3482,6 @@ static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev) mutex_destroy(&devr->srq_lock); } -static int -mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev) -{ - int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); - struct mlx5_core_dev *mdev = dev->mdev; - bool ro_supp = false; - void *mkc; - u32 mkey; - u32 pdn; - u32 *in; - int err; - - err = mlx5_core_alloc_pd(mdev, &pdn); - if (err) - return err; - - in = kvzalloc(inlen, GFP_KERNEL); - if (!in) { - err = -ENOMEM; - goto err; - } - - MLX5_SET(create_mkey_in, in, data_direct, 1); - mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); - MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); - MLX5_SET(mkc, mkc, lw, 1); - MLX5_SET(mkc, mkc, lr, 1); - MLX5_SET(mkc, mkc, rw, 1); - MLX5_SET(mkc, mkc, rr, 1); - MLX5_SET(mkc, mkc, a, 1); - MLX5_SET(mkc, mkc, pd, pdn); - MLX5_SET(mkc, mkc, length64, 1); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); - if (err) - goto err_mkey; - - dev->ddr.mkey = mkey; - dev->ddr.pdn = pdn; - - /* create another mkey with RO support */ - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) { - MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); - ro_supp = true; - } - - if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) { - MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); - ro_supp = true; - } - - if (ro_supp) { - err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); - /* RO is defined as best effort */ - if (!err) { - dev->ddr.mkey_ro = mkey; - dev->ddr.mkey_ro_valid = true; - } - } - - kvfree(in); - return 0; - -err_mkey: - kvfree(in); -err: - mlx5_core_dealloc_pd(mdev, pdn); - return err; -} - -static void -mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev) -{ - - if (dev->ddr.mkey_ro_valid) - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey_ro); - - mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey); - mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn); -} - static u32 get_core_cap_flags(struct ib_device *ibdev, struct mlx5_hca_vport_context *rep) { @@ -4016,39 +3947,42 @@ static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev, return false; } +static int mlx5_ib_data_direct_event(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct mlx5_ib_dev *dev = + container_of(nb, struct mlx5_ib_dev, data_direct_nb); + + if (action != MLX5_DATA_DIRECT_UNBIND) + return NOTIFY_DONE; + + mutex_lock(&dev->data_direct_lock); + mlx5_ib_revoke_data_direct_mrs(dev); + mutex_unlock(&dev->data_direct_lock); + + return NOTIFY_OK; +} + static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev) { - char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {}; int ret; - if (!MLX5_CAP_GEN(dev->mdev, data_direct) || - !MLX5_CAP_GEN_2(dev->mdev, query_vuid)) + if (!mlx5_data_direct_supported(dev->mdev)) return 0; - ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid); - if (ret) - return ret; - - ret = mlx5_ib_create_data_direct_resources(dev); - if (ret) - return ret; - INIT_LIST_HEAD(&dev->data_direct_mr_list); - ret = mlx5_data_direct_ib_reg(dev, vuid); - if (ret) - mlx5_ib_free_data_direct_resources(dev); + dev->data_direct_nb.notifier_call = mlx5_ib_data_direct_event; + ret = mlx5_data_direct_register(dev->mdev, &dev->data_direct_nb); return ret; } static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev) { - if (!MLX5_CAP_GEN(dev->mdev, data_direct) || - !MLX5_CAP_GEN_2(dev->mdev, query_vuid)) + if (!mlx5_data_direct_supported(dev->mdev)) return; - mlx5_data_direct_ib_unreg(dev); - mlx5_ib_free_data_direct_resources(dev); + mlx5_data_direct_unregister(dev->mdev, &dev->data_direct_nb); } static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev) @@ -5079,22 +5013,6 @@ static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev) cancel_work_sync(&devr->ports[port].pkey_change_work); } -void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev, - struct mlx5_data_direct_dev *dev) -{ - mutex_lock(&ibdev->data_direct_lock); - ibdev->data_direct_dev = dev; - mutex_unlock(&ibdev->data_direct_lock); -} - -void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev) -{ - mutex_lock(&ibdev->data_direct_lock); - mlx5_ib_revoke_data_direct_mrs(ibdev); - ibdev->data_direct_dev = NULL; - mutex_unlock(&ibdev->data_direct_lock); -} - void __mlx5_ib_remove(struct mlx5_ib_dev *dev, const struct mlx5_ib_profile *profile, int stage) @@ -5521,9 +5439,6 @@ static int __init mlx5_ib_init(void) ret = mlx5r_rep_init(); if (ret) goto rep_err; - ret = mlx5_data_direct_driver_register(); - if (ret) - goto dd_err; ret = auxiliary_driver_register(&mlx5r_mp_driver); if (ret) goto mp_err; @@ -5536,8 +5451,6 @@ static int __init mlx5_ib_init(void) drv_err: auxiliary_driver_unregister(&mlx5r_mp_driver); mp_err: - mlx5_data_direct_driver_unregister(); -dd_err: mlx5r_rep_cleanup(); rep_err: rcu_barrier(); @@ -5550,7 +5463,6 @@ static int __init mlx5_ib_init(void) static void __exit mlx5_ib_cleanup(void) { - mlx5_data_direct_driver_unregister(); auxiliary_driver_unregister(&mlx5r_driver); auxiliary_driver_unregister(&mlx5r_mp_driver); mlx5r_rep_cleanup(); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index e9ddf2e97a76..4a6ee14ffd7d 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -24,6 +24,7 @@ #include #include +#include #include "srq.h" #include "qp.h" #include "macsec.h" @@ -781,13 +782,6 @@ struct mlx5_ib_port_resources { struct work_struct pkey_change_work; }; -struct mlx5_data_direct_resources { - u32 pdn; - u32 mkey; - u32 mkey_ro; - u8 mkey_ro_valid :1; -}; - struct mlx5_ib_resources { struct ib_cq *c0; struct mutex cq_lock; @@ -1097,8 +1091,9 @@ struct mlx5_macsec { struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; - struct mlx5_data_direct_dev *data_direct_dev; - /* protect accessing data_direct_dev */ + /* Protects data_direct_mr_list and serializes mr + * registration/deregistration with data direct device unbind. + */ struct mutex data_direct_lock; struct notifier_block mdev_events; struct notifier_block sys_error_events; @@ -1131,6 +1126,7 @@ struct mlx5_ib_dev { spinlock_t reset_flow_resource_lock; struct list_head qp_list; struct list_head data_direct_mr_list; + struct notifier_block data_direct_nb; /* Array with num_ports elements */ struct mlx5_ib_port *port; struct mlx5_sq_bfreg bfreg; @@ -1155,7 +1151,6 @@ struct mlx5_ib_dev { u16 pkey_table_len; u8 lag_ports; struct mlx5_special_mkeys mkeys; - struct mlx5_data_direct_resources ddr; #ifdef CONFIG_MLX5_MACSEC struct mlx5_macsec macsec; @@ -1401,9 +1396,6 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table); struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, struct ib_dm_mr_attr *attr, struct uverbs_attr_bundle *attrs); -void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev, - struct mlx5_data_direct_dev *dev); -void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev); void mlx5_ib_revoke_data_direct_mrs(struct mlx5_ib_dev *dev); #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 00e13028762a..176f87ba062c 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -44,7 +44,6 @@ #include "dm.h" #include "mlx5_ib.h" #include "umr.h" -#include "data_direct.h" #include "dmah.h" static int mkey_max_umr_order(struct mlx5_ib_dev *dev) @@ -587,7 +586,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, populate ? pd : dev->umrc.pd); /* In case a data direct flow, overwrite the pdn field by its internal kernel PD */ if (umem->is_dmabuf && ksm_mode) - MLX5_SET(mkc, mkc, pd, dev->ddr.pdn); + MLX5_SET(mkc, mkc, pd, dev->mdev->data_direct->pdn); MLX5_SET(mkc, mkc, free, !populate); MLX5_SET(mkc, mkc, access_mode_1_0, access_mode); @@ -997,7 +996,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, return ERR_PTR(-EOPNOTSUPP); mutex_lock(&dev->data_direct_lock); - data_direct_dev = dev->data_direct_dev; + data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct); if (!data_direct_dev) { ret = -EINVAL; goto end; @@ -1006,7 +1005,7 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset, /* If no device's 'data direct mkey' with RO flags exists * mask it out accordingly. */ - if (!dev->ddr.mkey_ro_valid) + if (!dev->mdev->data_direct->mkey_ro_valid) access_flags &= ~IB_ACCESS_RELAXED_ORDERING; crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev, offset, length, virt_addr, fd, diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index b8618610737a..9c91e76029d4 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -300,7 +300,7 @@ static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, /* * Iteration one - zap the HW's MTTs. The notifiers_count ensures that * while we are doing the invalidation, no page fault will attempt to - * overwrite the same MTTs. Concurent invalidations might race us, + * overwrite the same MTTs. Concurrent invalidations might race us, * but they will write 0s as well, so no difference in the end result. */ for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { @@ -953,7 +953,7 @@ int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) int ret; if (mr->data_direct) - pdn = dev->ddr.pdn; + pdn = dev->mdev->data_direct->pdn; else pdn = to_mpd(pd)->pdn; @@ -1885,8 +1885,8 @@ mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) if (!eq->core) return 0; mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); - err = mlx5_eq_destroy_generic(dev->mdev, eq->core); cancel_work_sync(&eq->work); + err = mlx5_eq_destroy_generic(dev->mdev, eq->core); destroy_workqueue(eq->wq); mempool_destroy(eq->pool); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 0d4f8b109ad2..627dd17804c9 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -1624,6 +1624,8 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, if (err) return err; + sq->base.container_mibqp = qp; + sq->base.mqp.event = mlx5_ib_qp_event; err = create_raw_packet_qp_sq(dev, udata, attrs, sq, in, pd, to_mcq(init_attr->send_cq)); if (err) @@ -1635,9 +1637,6 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, resp->sqn = sq->base.mqp.qpn; resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN; } - - sq->base.container_mibqp = qp; - sq->base.mqp.event = mlx5_ib_qp_event; } if (qp->rq.wqe_cnt) { @@ -1647,6 +1646,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, } rq->base.container_mibqp = qp; + rq->base.mqp.event = mlx5_ib_qp_event; if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING) rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING; @@ -2092,13 +2092,13 @@ static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, } base = &qp->trans_qp.base; + base->container_mibqp = qp; + base->mqp.event = mlx5_ib_qp_event; err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out); kvfree(in); if (err) return err; - base->container_mibqp = qp; - base->mqp.event = mlx5_ib_qp_event; if (MLX5_CAP_GEN(mdev, ece_support)) params->resp.ece_options = MLX5_GET(create_qp_out, out, ece); @@ -2236,14 +2236,14 @@ static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING; } + base->container_mibqp = qp; + base->mqp.event = mlx5_ib_qp_event; err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out); kvfree(in); if (err) goto err_create; - base->container_mibqp = qp; - base->mqp.event = mlx5_ib_qp_event; if (MLX5_CAP_GEN(mdev, ece_support)) params->resp.ece_options = MLX5_GET(create_qp_out, out, ece); @@ -2432,6 +2432,8 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING; } + base->container_mibqp = qp; + base->mqp.event = mlx5_ib_qp_event; if (init_attr->qp_type == IB_QPT_RAW_PACKET || qp->flags & IB_QP_CREATE_SOURCE_QPN) { qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr; @@ -2446,8 +2448,6 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, if (err) goto err_create; - base->container_mibqp = qp; - base->mqp.event = mlx5_ib_qp_event; if (MLX5_CAP_GEN(mdev, ece_support)) params->resp.ece_options = MLX5_GET(create_qp_out, out, ece); @@ -2577,14 +2577,13 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, MLX5_CAP_GEN(mdev, go_back_n)) MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N); + base->container_mibqp = qp; + base->mqp.event = mlx5_ib_qp_event; err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out); kvfree(in); if (err) goto err_create; - base->container_mibqp = qp; - base->mqp.event = mlx5_ib_qp_event; - get_cqs(qp->type, attr->send_cq, attr->recv_cq, &send_cq, &recv_cq); spin_lock_irqsave(&dev->reset_flow_resource_lock, flags); @@ -3186,7 +3185,7 @@ static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, out: if (err) { - mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type); + mlx5_ib_err(dev, "Create QP type %d failed, err = %d\n", qp->type, err); return err; } @@ -5344,11 +5343,14 @@ static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type) break; default: mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn); - return; + goto out; } rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context); } + +out: + mlx5_core_res_put(&core_qp->common); } static int set_delay_drop(struct mlx5_ib_dev *dev) @@ -5616,6 +5618,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, rwq = kzalloc_obj(*rwq); if (!rwq) return ERR_PTR(-ENOMEM); + rwq->core_qp.event = mlx5_ib_wq_event; err = prepare_user_rq(pd, init_attr, udata, rwq); if (err) goto err; @@ -5639,7 +5642,6 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, goto err_copy; } - rwq->core_qp.event = mlx5_ib_wq_event; rwq->ibwq.event_handler = init_attr->event_handler; return &rwq->ibwq; diff --git a/drivers/infiniband/hw/mlx5/qpc.c b/drivers/infiniband/hw/mlx5/qpc.c index a7a4f9420271..ce2ef8828c09 100644 --- a/drivers/infiniband/hw/mlx5/qpc.c +++ b/drivers/infiniband/hw/mlx5/qpc.c @@ -128,7 +128,7 @@ static int rsc_event_notifier(struct notifier_block *nb, case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; - rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN); + rsn |= ((u32)eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN); break; default: return NOTIFY_DONE; @@ -146,6 +146,8 @@ static int rsc_event_notifier(struct notifier_block *nb, case MLX5_RES_RQ: case MLX5_RES_SQ: qp = (struct mlx5_core_qp *)common; + if (WARN_ON_ONCE(!qp->event)) + goto out; qp->event(qp, event_type); /* Need to put resource in event handler */ return NOTIFY_OK; @@ -165,6 +167,10 @@ static int create_resource_common(struct mlx5_ib_dev *dev, int err; qp->common.res = rsc_type; + refcount_set(&qp->common.refcount, 1); + init_completion(&qp->common.free); + qp->pid = current->pid; + spin_lock_irq(&table->lock); err = radix_tree_insert(&table->tree, qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN), @@ -173,10 +179,6 @@ static int create_resource_common(struct mlx5_ib_dev *dev, if (err) return err; - refcount_set(&qp->common.refcount, 1); - init_completion(&qp->common.free); - qp->pid = current->pid; - return 0; } @@ -258,6 +260,14 @@ int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp, qp->uid = MLX5_GET(create_qp_in, in, uid); qp->qpn = MLX5_GET(create_qp_out, out, qpn); + /* Set ibqp.qp_num before create_resource_common() inserts the QP into + * the radix tree and makes it visible to EQE processing. The + * assignment in create_qp() happens only after this function returns, + * leaving a window where an arriving EQE would observe qp_num == 0. + * create_qp() still overrides this with 0/1 for QP0/QP1, and sets it + * independently for DCT (which does not go through this function). + */ + to_mibqp(qp)->ibqp.qp_num = qp->qpn; err = create_resource_common(dev, qp, MLX5_RES_QP); if (err) diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c index a973c1b7515f..6a4a937d97ce 100644 --- a/drivers/infiniband/hw/mlx5/srq.c +++ b/drivers/infiniband/hw/mlx5/srq.c @@ -284,6 +284,7 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, in.pd = to_mpd(ib_srq->pd)->pdn; in.db_record = srq->db.dma; + srq->msrq.event = mlx5_ib_srq_event; err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); kvfree(in.pas); if (err) { @@ -292,8 +293,6 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq, } mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); - - srq->msrq.event = mlx5_ib_srq_event; srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; if (udata) { diff --git a/drivers/infiniband/hw/mlx5/std_types.c b/drivers/infiniband/hw/mlx5/std_types.c index 1ee31611b4b3..b705b88dc917 100644 --- a/drivers/infiniband/hw/mlx5/std_types.c +++ b/drivers/infiniband/hw/mlx5/std_types.c @@ -10,7 +10,6 @@ #include #include #include "mlx5_ib.h" -#include "data_direct.h" #define UVERBS_MODULE_NAME mlx5_ib #include @@ -203,7 +202,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_GET_DATA_DIRECT_SYSFS_PATH)( return PTR_ERR(c); dev = to_mdev(c->ibucontext.device); mutex_lock(&dev->data_direct_lock); - data_direct_dev = dev->data_direct_dev; + data_direct_dev = mlx5_data_direct_get_dev(dev->mdev->data_direct); if (!data_direct_dev) { ret = -ENODEV; goto end; diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c index 951de1d85632..711075e3d9ea 100644 --- a/drivers/infiniband/hw/mlx5/umr.c +++ b/drivers/infiniband/hw/mlx5/umr.c @@ -759,12 +759,14 @@ static int _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, } if (dd) { + struct mlx5_data_direct *mdd = dev->mdev->data_direct; + cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter)); if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING && - dev->ddr.mkey_ro_valid) - cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro); + mdd->mkey_ro_valid) + cur_ksm->key = cpu_to_be32(mdd->mkey_ro); else - cur_ksm->key = cpu_to_be32(dev->ddr.mkey); + cur_ksm->key = cpu_to_be32(mdd->mkey); if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP)) { cur_ksm->va = 0; @@ -809,7 +811,8 @@ int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) return -EINVAL; - return _mlx5r_umr_update_mr_pas(mr, flags, mr_to_mdev(mr)->ddr.pdn, + return _mlx5r_umr_update_mr_pas(mr, flags, + mr_to_mdev(mr)->mdev->data_direct->pdn, true, start_block, nblocks); } diff --git a/drivers/infiniband/hw/mlx5/wr.c b/drivers/infiniband/hw/mlx5/wr.c index fca9e1d9d5e9..bb2c60380fe2 100644 --- a/drivers/infiniband/hw/mlx5/wr.c +++ b/drivers/infiniband/hw/mlx5/wr.c @@ -927,6 +927,7 @@ static noinline_for_stack int handle_reg_mr_integrity(struct mlx5_ib_dev *dev, qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; out: + mr->pi_mr = pi_mr; return err; } diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c index 26c3408dcaca..39d4ff15849e 100644 --- a/drivers/infiniband/hw/mthca/mthca_cq.c +++ b/drivers/infiniband/hw/mthca/mthca_cq.c @@ -300,7 +300,8 @@ void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, * Now sweep backwards through the CQ, removing CQ entries * that match our QP by copying older entries on top of them. */ - while ((int) --prod_index - (int) cq->cons_index >= 0) { + while (prod_index != cq->cons_index) { + --prod_index; cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); if (cqe->my_qpn == cpu_to_be32(qpn)) { if (srq && is_recv_cqe(cqe)) diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index cfe3d19b73b3..97c2e18e67ad 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -1392,9 +1392,9 @@ int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, qp = get_ocrdma_qp(ibqp); dev = get_ocrdma_dev(ibqp->device); - /* syncronize with multiple context trying to change, retrive qps */ + /* synchronize with multiple context trying to change, retrieve qps */ mutex_lock(&dev->dev_lock); - /* syncronize with wqe, rqe posting and cqe processing contexts */ + /* synchronize with wqe, rqe posting and cqe processing contexts */ spin_lock_irqsave(&qp->q_lock, flags); old_qps = get_ibqp_state(qp->state); if (attr_mask & IB_QP_STATE) @@ -1599,7 +1599,7 @@ static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq) */ cur_getp = cq->getp; - /* find upto when do we reap the cq. */ + /* find up to when do we reap the cq. */ stop_getp = cur_getp; do { if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp))) diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 012a0ab98d6b..acab3cb98caa 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -775,16 +775,23 @@ static inline int qedr_init_user_queue(struct ib_udata *udata, struct qedr_userq *q, u64 buf_addr, size_t buf_len, bool requires_db_rec, int access, - int alloc_and_init) + int alloc_and_init, bool is_cq) { u32 fw_pages; int rc; q->buf_addr = buf_addr; q->buf_len = buf_len; - q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, access); + if (is_cq) + q->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL, + q->buf_addr, q->buf_len, + access); + else + q->umem = ib_umem_get_va(&dev->ibdev, q->buf_addr, q->buf_len, + access); if (IS_ERR(q->umem)) { - DP_ERR(dev, "create user queue: failed ib_umem_get_va, got %ld\n", + DP_ERR(dev, "create user queue: failed %s, got %ld\n", + is_cq ? "ib_umem_get_cq_buf_or_va" : "ib_umem_get_va", PTR_ERR(q->umem)); return PTR_ERR(q->umem); } @@ -946,7 +953,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, ureq.len, true, IB_ACCESS_LOCAL_WRITE, - 1); + 1, true); if (rc) goto err0; @@ -1075,7 +1082,7 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) /* We don't want the IRQ handler to handle a non-existing CQ so we * wait until all CNQ interrupts, if any, are received. This will always * happen and will always happen very fast. If not, then a serious error - * has occured. That is why we can use a long delay. + * has occurred. That is why we can use a long delay. * We spin for a short time so we don’t lose time on context switching * in case all the completions are handled in that span. Otherwise * we sleep for a while and check again. Since the CNQ may be @@ -1438,7 +1445,7 @@ static int qedr_init_srq_user_params(struct ib_udata *udata, int rc; rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, - ureq->srq_len, false, access, 1); + ureq->srq_len, false, access, 1, false); if (rc) return rc; @@ -1831,7 +1838,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev, if (qedr_qp_has_sq(qp)) { /* SQ - read access only (0) */ rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, - ureq.sq_len, true, 0, alloc_and_init); + ureq.sq_len, true, 0, alloc_and_init, + false); if (rc) return rc; } @@ -1839,7 +1847,8 @@ static int qedr_create_user_qp(struct qedr_dev *dev, if (qedr_qp_has_rq(qp)) { /* RQ - read access only (0) */ rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, - ureq.rq_len, true, 0, alloc_and_init); + ureq.rq_len, true, 0, alloc_and_init, + false); if (rc) { ib_umem_release(qp->usq.umem); qp->usq.umem = NULL; diff --git a/drivers/infiniband/hw/usnic/usnic_abi.h b/drivers/infiniband/hw/usnic/usnic_abi.h index 86a82a4da0aa..a684bc54f3d4 100644 --- a/drivers/infiniband/hw/usnic/usnic_abi.h +++ b/drivers/infiniband/hw/usnic/usnic_abi.h @@ -72,7 +72,7 @@ struct usnic_ib_create_qp_resp { u64 bar_bus_addr; u32 bar_len; /* - * WQ, RQ, CQ are explicitly specified bc exposing a generic resources inteface + * WQ, RQ, CQ are explicitly specified bc exposing a generic resources interface * expands the scope of ABI to many files. */ u32 wq_cnt; diff --git a/drivers/infiniband/hw/usnic/usnic_debugfs.c b/drivers/infiniband/hw/usnic/usnic_debugfs.c index 10a8cd5ba076..ece7c6e71499 100644 --- a/drivers/infiniband/hw/usnic/usnic_debugfs.c +++ b/drivers/infiniband/hw/usnic/usnic_debugfs.c @@ -76,7 +76,7 @@ static ssize_t flowinfo_read(struct file *f, char __user *data, qp_flow = f->private_data; ptr = buf; - left = count; + left = sizeof(buf); if (*ppos > 0) return 0; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index 1a1647d0e345..ebba3027cf48 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -181,7 +181,7 @@ find_free_vf_and_create_qp_grp(struct ib_qp *qp, } if (usnic_ib_share_vf) { - /* Try to find resouces on a used vf which is in pd */ + /* Try to find resources on a used vf which is in pd */ dev_list = usnic_uiom_get_dev_list(pd->umem_pd); if (IS_ERR(dev_list)) return PTR_ERR(dev_list); diff --git a/drivers/infiniband/hw/usnic/usnic_transport.h b/drivers/infiniband/hw/usnic/usnic_transport.h index 9a7a2d9755c0..c963c92218d3 100644 --- a/drivers/infiniband/hw/usnic/usnic_transport.h +++ b/drivers/infiniband/hw/usnic/usnic_transport.h @@ -50,7 +50,7 @@ int usnic_transport_sock_to_str(char *buf, int buf_sz, u16 usnic_transport_rsrv_port(enum usnic_transport_type type, u16 port_num); void usnic_transport_unrsrv_port(enum usnic_transport_type type, u16 port_num); /* - * Do a fget on the socket refered to by sock_fd and returns the socket. + * Do a fget on the socket referred to by sock_fd and returns the socket. * Socket will not be destroyed before usnic_transport_put_socket has * been called. */ diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index e939cd5ce40b..53cc49f2b8c9 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -270,7 +270,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, /* set qp->sq.wqe_cnt, shift, buf_size.. */ qp->rumem = ib_umem_get_va(ibqp->device, ucmd.rbuf_addr, - ucmd.rbuf_size, 0); + ucmd.rbuf_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(qp->rumem)) { ret = PTR_ERR(qp->rumem); goto err_qp; @@ -282,7 +283,8 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, } qp->sumem = ib_umem_get_va(ibqp->device, ucmd.sbuf_addr, - ucmd.sbuf_size, 0); + ucmd.sbuf_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(qp->sumem)) { if (!is_srq) ib_umem_release(qp->rumem); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c index 345ec486a223..3252c2eb405a 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c @@ -146,7 +146,8 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, if (ret) goto err_srq; - srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, 0); + srq->umem = ib_umem_get_va(ibsrq->device, ucmd.buf_addr, ucmd.buf_size, + IB_ACCESS_LOCAL_WRITE); if (IS_ERR(srq->umem)) { ret = PTR_ERR(srq->umem); goto err_srq; diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index c40cce69e945..3602b9ff2196 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -814,7 +814,7 @@ int rvt_alloc_rq(struct rvt_rq *rq, u32 size, int node, * * This function is called from both rvt_create_qp() and * rvt_reset_qp(). The difference is that the reset - * patch the necessary locks to protect against concurent + * patch the necessary locks to protect against concurrent * access. */ static void rvt_init_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, @@ -1173,7 +1173,7 @@ int rvt_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr, if (rdi->driver_f.qp_priv_init) { ret = rdi->driver_f.qp_priv_init(rdi, qp, init_attr); if (ret) - goto bail_rq_wq; + goto bail_qpn; } break; diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c index d022aa56c5bf..8122b46ed4ab 100644 --- a/drivers/infiniband/sw/rdmavt/srq.c +++ b/drivers/infiniband/sw/rdmavt/srq.c @@ -286,7 +286,7 @@ int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) } /** - * rvt_destroy_srq - destory an srq + * rvt_destroy_srq - destroy an srq * @ibsrq: srq object to destroy * @udata: user data for libibverbs.so */ diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h index 64d636bf80fd..9a8e96dfb792 100644 --- a/drivers/infiniband/sw/rxe/rxe_loc.h +++ b/drivers/infiniband/sw/rxe/rxe_loc.h @@ -64,7 +64,7 @@ int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length); int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, unsigned int length, enum rxe_mr_copy_dir dir); int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma, - void *addr, int length, enum rxe_mr_copy_dir dir); + void *addr, u32 length, enum rxe_mr_copy_dir dir); int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); enum resp_states rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, @@ -91,7 +91,8 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem); /* rxe_net.c */ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, - int paylen, struct rxe_pkt_info *pkt); + u32 paylen, struct rxe_pkt_info *pkt); +void rxe_put_skb(struct sk_buff *skb); int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt, struct sk_buff *skb); int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, @@ -184,6 +185,9 @@ void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb); static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp) { + if (unlikely(opcode < 0 || opcode > IB_WR_REG_MR)) + return 0; + return rxe_wr_opcode_info[opcode].mask[qp->ibqp.qp_type]; } diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c index 71d9ea477289..13cb28d3916f 100644 --- a/drivers/infiniband/sw/rxe/rxe_mr.c +++ b/drivers/infiniband/sw/rxe/rxe_mr.c @@ -418,13 +418,13 @@ int copy_data( int access, struct rxe_dma_info *dma, void *addr, - int length, + u32 length, enum rxe_mr_copy_dir dir) { - int bytes; + u32 bytes; struct rxe_sge *sge = &dma->sge[dma->cur_sge]; - int offset = dma->sge_offset; - int resid = dma->resid; + u32 offset = dma->sge_offset; + u32 resid = dma->resid; struct rxe_mr *mr = NULL; u64 iova; int err; @@ -777,6 +777,10 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe) u32 key = wqe->wr.wr.reg.key; u32 access = wqe->wr.wr.reg.access; + /* IB_WR_REG_MR is a kernel-only op; a user QP must not reach here. */ + if (qp->is_user) + return -EOPNOTSUPP; + /* user can only register MR in free state */ if (unlikely(mr->state != RXE_MR_STATE_FREE)) { rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n", mr->lkey); @@ -796,6 +800,12 @@ int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe) return -EINVAL; } + /* an MR with no umem is never an ODP MR */ + if (unlikely(access & IB_ACCESS_ON_DEMAND)) { + rxe_dbg_mr(mr, "access = 0x%x requests ODP\n", access); + return -EINVAL; + } + mr->access = access; mr->lkey = key; mr->rkey = key; diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c index bddb7a257831..a2e3a6ce0946 100644 --- a/drivers/infiniband/sw/rxe/rxe_mw.c +++ b/drivers/infiniband/sw/rxe/rxe_mw.c @@ -161,6 +161,7 @@ static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe, if (mw->ibmw.type == IB_MW_TYPE_2) { rxe_get(qp); + atomic_inc(&qp->mw_bind_num); mw->qp = qp; } } @@ -245,6 +246,7 @@ static void rxe_do_invalidate_mw(struct rxe_mw *mw) /* valid type 2 MW will always have a QP pointer */ qp = mw->qp; mw->qp = NULL; + atomic_dec(&qp->mw_bind_num); rxe_put(qp); /* valid type 2 MW will always have an MR pointer */ @@ -332,6 +334,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem) struct rxe_qp *qp = mw->qp; mw->qp = NULL; + atomic_dec(&qp->mw_bind_num); rxe_put(qp); } diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c index 53daaf4c1eb2..c572cda8fe18 100644 --- a/drivers/infiniband/sw/rxe/rxe_net.c +++ b/drivers/infiniband/sw/rxe/rxe_net.c @@ -429,6 +429,29 @@ int rxe_prepare(struct rxe_av *av, struct rxe_pkt_info *pkt, return err; } +/* + * skb->dev may be rewritten on the TX path (VLAN/bond/tunnel, etc.). + * The netdev we held in rxe_init_packet() is kept in destructor_arg so + * that put always matches hold. skb->cb cannot be used: it is already + * occupied by struct rxe_pkt_info and rewritten by IP control blocks. + */ +static void rxe_skb_set_held_ndev(struct sk_buff *skb, struct net_device *ndev) +{ + dev_hold(ndev); + skb->dev = ndev; + skb_shinfo(skb)->destructor_arg = ndev; +} + +static void rxe_skb_put_held_ndev(struct sk_buff *skb) +{ + struct net_device *ndev = skb_shinfo(skb)->destructor_arg; + + if (ndev) { + skb_shinfo(skb)->destructor_arg = NULL; + dev_put(ndev); + } +} + static void rxe_skb_tx_dtor(struct sk_buff *skb) { struct rxe_qp *qp = skb->sk->sk_user_data; @@ -441,6 +464,18 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb) rxe_put(qp); sock_put(skb->sk); + rxe_skb_put_held_ndev(skb); +} + +/* + * Free an skb that still holds a netdev reference from rxe_init_packet() + * and does not yet have rxe_skb_tx_dtor() installed. Once the TX + * destructor is set, callers must use kfree_skb() instead. + */ +void rxe_put_skb(struct sk_buff *skb) +{ + rxe_skb_put_held_ndev(skb); + kfree_skb(skb); } static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt) @@ -529,14 +564,14 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt, goto done; drop: - kfree_skb(skb); + rxe_put_skb(skb); err = 0; done: return err; } struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, - int paylen, struct rxe_pkt_info *pkt) + u32 paylen, struct rxe_pkt_info *pkt) { unsigned int hdr_len; struct sk_buff *skb = NULL; @@ -574,8 +609,7 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, skb_reserve(skb, hdr_len + LL_RESERVED_SPACE(ndev)); - /* FIXME: hold reference to this netdev until life of this skb. */ - skb->dev = ndev; + rxe_skb_set_held_ndev(skb, ndev); rcu_read_unlock(); if (av->network_type == RXE_NETWORK_TYPE_IPV4) @@ -638,9 +672,8 @@ static void rxe_sock_put(struct sock *sk, if (refcount_read(&sk->sk_refcnt) > SK_REF_FOR_TUNNEL) { __sock_put(sk); } else { + set_sk(net, NULL); rxe_release_udp_tunnel(sk); - sk = NULL; - set_sk(net, sk); } } @@ -710,6 +743,28 @@ void rxe_set_port_state(struct rxe_dev *rxe) dev_put(ndev); } +/* + * Move all QPs to the error state so pending send/recv work is drained and + * in-flight TX skbs (which hold a netdev reference) can be released. Called + * from the netdev notifier so unregister cannot stall on held skbs. + */ +static void rxe_flush_qps(struct rxe_dev *rxe) +{ + struct rxe_pool_elem *elem; + struct rxe_qp *qp; + unsigned long index; + + rcu_read_lock(); + xa_for_each(&rxe->qp_pool.xa, index, elem) { + if (!elem || !kref_get_unless_zero(&elem->ref_cnt)) + continue; + qp = elem->obj; + rxe_qp_error(qp); + rxe_put(qp); + } + rcu_read_unlock(); +} + static int rxe_notify(struct notifier_block *not_blk, unsigned long event, void *arg) @@ -721,7 +776,12 @@ static int rxe_notify(struct notifier_block *not_blk, return NOTIFY_OK; switch (event) { + case NETDEV_GOING_DOWN: + /* Start draining TX queues before the netdev disappears. */ + rxe_flush_qps(rxe); + break; case NETDEV_UNREGISTER: + rxe_flush_qps(rxe); ib_unregister_device_queued(&rxe->ib_dev); rxe_net_del(&rxe->ib_dev); break; @@ -735,7 +795,6 @@ static int rxe_notify(struct notifier_block *not_blk, rxe_counter_inc(rxe, RXE_CNT_LINK_DOWNED); break; case NETDEV_REBOOT: - case NETDEV_GOING_DOWN: case NETDEV_CHANGEADDR: case NETDEV_CHANGENAME: case NETDEV_FEAT_CHANGE: diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c index ab21b620e94c..9d27244f89bc 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -136,6 +136,11 @@ static inline bool rxe_check_pagefault(struct ib_umem_odp *umem_odp, u64 iova, while (addr < iova + length) { idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; + if (idx >= ib_umem_odp_num_pages(umem_odp)) { + need_fault = true; + break; + } + if ((umem_odp->map.pfn_list[idx] & access) != access) { need_fault = true; break; @@ -469,7 +474,7 @@ static int rxe_ib_prefetch_sg_list(struct ib_pd *ibpd, struct rxe_mr *mr; struct ib_umem_odp *umem_odp; - mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE, + mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE | IB_ACCESS_ON_DEMAND, sg_list[i].lkey, RXE_LOOKUP_LOCAL); if (!mr) { @@ -535,7 +540,7 @@ static int rxe_ib_advise_mr_prefetch(struct ib_pd *ibpd, for (i = 0; i < num_sge; ++i) { /* Takes a reference, which will be released in the queued work */ - mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE, + mr = lookup_mr(pd, IB_ACCESS_LOCAL_WRITE | IB_ACCESS_ON_DEMAND, sg_list[i].lkey, RXE_LOOKUP_LOCAL); if (!mr) { mr = ERR_PTR(-EINVAL); diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index 311f285d78a6..fea55bc3b0cb 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -858,6 +858,16 @@ int rxe_qp_chk_destroy(struct rxe_qp *qp) return -EBUSY; } + /* An attempt to destroy a QP while it still holds references for + * bound type 2 memory windows will fail immediately. Otherwise the + * QP is freed with those references outstanding and the windows are + * left pointing at freed memory (use-after-free). + */ + if (atomic_read(&qp->mw_bind_num)) { + rxe_dbg_qp(qp, "Attempt to destroy while type 2 MWs are bound\n"); + return -EBUSY; + } + return 0; } diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index 2c90387aa126..e1d90633f085 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -320,6 +320,14 @@ void rxe_rcv(struct sk_buff *skb) struct rxe_dev *rxe = pkt->rxe; unsigned int skblen = skb->len - skb_network_offset(skb); + /* + * pkt lives in skb->cb, which the network stack has already used + * for its own state: for IPv6 the qword holding pkt->qp holds + * inet6_skb_parm.nhoff. The drop label puts pkt->qp, so clear + * it before the first early exit. + */ + pkt->qp = NULL; + if (unlikely(skb->len < RXE_BTH_BYTES)) goto drop; @@ -328,7 +336,6 @@ void rxe_rcv(struct sk_buff *skb) pkt->opcode = bth_opcode(pkt); pkt->psn = bth_psn(pkt); - pkt->qp = NULL; pkt->mask |= rxe_opcode[pkt->opcode].mask; /* diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 24f5c044363f..4f353abd76f5 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -510,7 +510,14 @@ static int finish_packet(struct rxe_qp *qp, struct rxe_av *av, if (pkt->mask & RXE_WRITE_OR_SEND_MASK) { if (wqe->wr.send_flags & IB_SEND_INLINE) { - u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset]; + u32 sge_offset = READ_ONCE(wqe->dma.sge_offset); + u8 *tmp; + + if (unlikely(sge_offset > (u32)qp->sq.max_inline || + payload > (u32)qp->sq.max_inline - sge_offset)) + return -EINVAL; + + tmp = &wqe->dma.inline_data[sge_offset]; memcpy(payload_addr(pkt), tmp, payload); @@ -811,7 +818,7 @@ int rxe_requester(struct rxe_qp *qp) wqe->status = IB_WC_LOC_PROT_ERR; else wqe->status = IB_WC_LOC_QP_OP_ERR; - kfree_skb(skb); + rxe_put_skb(skb); if (ah) rxe_put(ah); goto err; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 02b16e2b49b8..1b7f31737225 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -288,6 +288,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp) } size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); memcpy(&qp->resp.srq_wqe, wqe, size); + qp->resp.srq_wqe.wqe.dma.num_sge = num_sge; qp->resp.wqe = &qp->resp.srq_wqe.wqe; queue_advance_consumer(q, QUEUE_TYPE_FROM_CLIENT); @@ -328,6 +329,7 @@ static enum resp_states rxe_get_recv_wqe(struct rxe_qp *qp) } size = sizeof(*wqe) + num_sge * sizeof(struct rxe_sge); memcpy(&qp->resp.srq_wqe, wqe, size); + qp->resp.srq_wqe.wqe.dma.num_sge = num_sge; qp->resp.wqe = &qp->resp.srq_wqe.wqe; return RESPST_CHK_LENGTH; @@ -612,7 +614,7 @@ static enum resp_states check_rkey(struct rxe_qp *qp, } static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr, - int data_len) + u32 data_len) { int err; @@ -827,20 +829,21 @@ static enum resp_states atomic_write_reply(struct rxe_qp *qp, static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, struct rxe_pkt_info *ack, int opcode, - int payload, + u32 payload, u32 psn, u8 syndrome) { struct rxe_dev *rxe = to_rdev(qp->ibqp.device); struct sk_buff *skb; - int paylen; - int pad; + u32 paylen; + u32 pad = 0; int err; /* * allocate packet */ - pad = (-payload) & 0x3; + if (payload % 4) + pad = 4 - payload % 4; paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE; skb = rxe_init_packet(rxe, &qp->pri_av, paylen, ack); @@ -866,7 +869,7 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, err = rxe_prepare(&qp->pri_av, ack, skb); if (err) { - kfree_skb(skb); + rxe_put_skb(skb); return NULL; } @@ -934,9 +937,9 @@ static enum resp_states read_reply(struct rxe_qp *qp, { struct rxe_pkt_info ack_pkt; struct sk_buff *skb; - int mtu = qp->mtu; + u32 mtu = qp->mtu; enum resp_states state; - int payload; + u32 payload; int opcode; int err; struct resp_res *res = qp->resp.res; @@ -982,7 +985,7 @@ static enum resp_states read_reply(struct rxe_qp *qp, res->state = rdatm_res_state_next; - payload = min_t(int, res->read.resid, mtu); + payload = min(res->read.resid, mtu); skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload, res->cur_psn, AETH_ACK_UNLIMITED); @@ -994,7 +997,7 @@ static enum resp_states read_reply(struct rxe_qp *qp, err = rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt), payload, RXE_FROM_MR_OBJ); if (err) { - kfree_skb(skb); + rxe_put_skb(skb); state = RESPST_ERR_RKEY_VIOLATION; goto err_out; } diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c index 3864284522eb..46d0810ea2a7 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.c +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c @@ -1337,6 +1337,12 @@ static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags, return ERR_PTR(-EOPNOTSUPP); } + if ((flags & IB_MR_REREG_ACCESS) && + ((access ^ mr->access) & IB_ACCESS_ON_DEMAND)) { + rxe_err_mr(mr, "cannot change IB_ACCESS_ON_DEMAND\n"); + return ERR_PTR(-EOPNOTSUPP); + } + if (flags & IB_MR_REREG_PD) { rxe_put(old_pd); rxe_get(pd); diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h index 0f5ffd94643f..80df376e61ee 100644 --- a/drivers/infiniband/sw/rxe/rxe_verbs.h +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h @@ -261,6 +261,7 @@ struct rxe_qp { struct rxe_av alt_av; atomic_t mcg_num; + atomic_t mw_bind_num; struct sk_buff_head req_pkts; struct sk_buff_head resp_pkts; diff --git a/drivers/infiniband/sw/siw/siw_main.c b/drivers/infiniband/sw/siw/siw_main.c index 9735b75ac933..c08796e4262a 100644 --- a/drivers/infiniband/sw/siw/siw_main.c +++ b/drivers/infiniband/sw/siw/siw_main.c @@ -217,8 +217,11 @@ static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id) /* * siw_qp_id2obj() increments object reference count */ + struct ib_qp *base_qp = &qp->base_qp; + siw_qp_put(qp); - return &qp->base_qp; + + return base_qp; } return NULL; } diff --git a/drivers/infiniband/sw/siw/siw_qp_tx.c b/drivers/infiniband/sw/siw/siw_qp_tx.c index f7dd32c6e5ba..2675bc0b2019 100644 --- a/drivers/infiniband/sw/siw/siw_qp_tx.c +++ b/drivers/infiniband/sw/siw/siw_qp_tx.c @@ -85,7 +85,7 @@ static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr) if (likely(PAGE_SIZE - off >= bytes)) { memcpy(paddr, buffer + off, bytes); } else { - unsigned long part = bytes - (PAGE_SIZE - off); + unsigned long part = PAGE_SIZE - off; memcpy(paddr, buffer + off, part); kunmap_local(buffer); @@ -336,7 +336,6 @@ static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset, if (size + offset <= PAGE_SIZE) msg.msg_flags &= ~MSG_MORE; - tcp_rate_check_app_limited(sk); if (!sendpage_ok(page[i])) msg.msg_flags &= ~MSG_SPLICE_PAGES; bvec_set_page(&bvec, page[i], bytes, offset); @@ -805,17 +804,21 @@ static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe) if (wqe->wr_status == SIW_WR_QUEUED) { if (!(wqe->sqe.flags & SIW_WQE_INLINE)) { - if (tx_type(wqe) == SIW_OP_READ_RESPONSE) - wqe->sqe.num_sge = 1; - if (tx_type(wqe) != SIW_OP_READ && tx_type(wqe) != SIW_OP_READ_LOCAL_INV) { + enum ib_access_flags perms = 0; + /* - * Reference memory to be tx'd w/o checking - * access for LOCAL_READ permission, since - * not defined in RDMA core. + * Local READ permission doesn't exist in RDMA core. + * A READ RESPONSE source is read by the remote peer, + * so it must be checked for REMOTE_READ permission. */ - rv = siw_check_sgl_tx(qp->pd, wqe, 0); + if (tx_type(wqe) == SIW_OP_READ_RESPONSE) { + perms = IB_ACCESS_REMOTE_READ; + wqe->sqe.num_sge = 1; + } + + rv = siw_check_sgl_tx(qp->pd, wqe, perms); if (rv < 0) { if (tx_type(wqe) == SIW_OP_READ_RESPONSE) diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 6c14246befb1..f0b72d65fdeb 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -230,6 +230,13 @@ static int ipoib_stop(struct net_device *dev) clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags); + /* Normal flush runs without RTNL and may bring the device back + * up. Cancel it before the device down transition. + * Heavy flush is already serialized by RTNL and must not be + * waited on here while RTNL is held. + */ + cancel_work_sync(&priv->flush_normal); + netif_stop_queue(dev); ipoib_ib_dev_down(dev); diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c index ba3b45152907..697cad0d063d 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c @@ -140,7 +140,7 @@ int __ipoib_vlan_add(struct ipoib_dev_priv *ppriv, struct ipoib_dev_priv *priv, goto out_early; } - /* RTNL childs don't need proprietary sysfs entries */ + /* RTNL children don't need proprietary sysfs entries */ if (type == IPOIB_LEGACY_CHILD) { if (ipoib_cm_add_mode_attr(ndev)) goto sysfs_failed; diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c index 7df441685780..31e3a8ab6c48 100644 --- a/drivers/infiniband/ulp/iser/iscsi_iser.c +++ b/drivers/infiniband/ulp/iser/iscsi_iser.c @@ -498,7 +498,7 @@ static int iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session, * iscsi_iser_conn_start() - start iscsi-iser connection * @cls_conn: iscsi class connection * - * Notes: Here iser intialize (or re-initialize) stop_completion as + * Notes: Here iser initialize (or re-initialize) stop_completion as * from this point iscsi must call conn_stop in session/connection * teardown so iser transport must wait for it. */ @@ -593,7 +593,7 @@ static inline unsigned int iser_dif_prot_caps(int prot_caps) * @qdepth: session command queue depth * @initial_cmdsn: initiator command sequnce number * - * Allocates and adds a scsi host, expose DIF supprot if + * Allocates and adds a scsi host, expose DIF support if * exists, and sets up an iscsi session. */ static struct iscsi_cls_session * @@ -866,7 +866,7 @@ static int iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) * @ep: iscsi endpoint handle * * This routine is not blocked by iser and RDMA termination process - * completion as we queue a deffered work for iser/RDMA destruction + * completion as we queue a deferred work for iser/RDMA destruction * and cleanup or actually call it immediately in case we didn't pass * iscsi conn bind/start stage, thus it is safe. */ diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c index 55fe68e5b837..ab2a27d23478 100644 --- a/drivers/infiniband/ulp/iser/iser_verbs.c +++ b/drivers/infiniband/ulp/iser/iser_verbs.c @@ -347,7 +347,7 @@ void iser_release_work(struct work_struct *work) /* Wait for conn_stop to complete */ wait_for_completion(&iser_conn->stop_completion); - /* Wait for IB resouces cleanup to complete */ + /* Wait for IB resources cleanup to complete */ wait_for_completion(&iser_conn->ib_completion); mutex_lock(&iser_conn->state_mutex); diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index e69db43370ff..efa8d7b7844c 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -2583,7 +2583,7 @@ static void isert_put_unsol_pending_cmds(struct iscsit_conn *conn) { struct iscsit_cmd *cmd, *tmp; - static LIST_HEAD(drop_cmd_list); + LIST_HEAD(drop_cmd_list); spin_lock_bh(&conn->cmd_lock); list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c index eac38b57b00d..bf3a0377da3f 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c @@ -70,19 +70,24 @@ __rtrs_get_permit(struct rtrs_clt_sess *clt, enum rtrs_clt_con_type con_type) { size_t max_depth = clt->queue_depth; struct rtrs_permit *permit; - int bit; + unsigned long bit = 0; /* - * Adapted from null_blk get_tag(). Callers from different cpus may - * grab the same bit, since find_first_zero_bit is not atomic. - * But then the test_and_set_bit_lock will fail for all the - * callers but one, so that they will loop again. - * This way an explicit spinlock is not required. + * Callers from different CPUs may grab the same bit, since the bitmap + * scan is not atomic. But then the test_and_set_bit_lock() will fail + * for all the callers but one, so that they loop again. This way an + * explicit spinlock is not required. find_next_zero_bit() resumes + * from the last position so that a lost race does not rescan the + * already-set low bits; if it reaches the end, wrap to the beginning + * to exhaust the map and still find a permit freed below the cursor. */ do { - bit = find_first_zero_bit(clt->permits_map, max_depth); - if (bit >= max_depth) - return NULL; + bit = find_next_zero_bit(clt->permits_map, max_depth, bit); + if (bit >= max_depth) { + bit = find_first_zero_bit(clt->permits_map, max_depth); + if (bit >= max_depth) + return NULL; + } } while (test_and_set_bit_lock(bit, clt->permits_map)); permit = get_permit(clt, bit); @@ -489,18 +494,22 @@ static int rtrs_post_send_rdma(struct rtrs_clt_con *con, imm, flags, wr, NULL); } -static void process_io_rsp(struct rtrs_clt_path *clt_path, u32 msg_id, +static bool process_io_rsp(struct rtrs_clt_path *clt_path, u32 msg_id, s16 errno, bool w_inval) { struct rtrs_clt_io_req *req; - if (WARN_ON(msg_id >= clt_path->queue_depth)) - return; + if (!clt_path->reqs || msg_id >= clt_path->queue_depth) + return false; req = &clt_path->reqs[msg_id]; + if (!req->mr) + return false; + /* Drop need_inv if server responded with send with invalidation */ req->mr->need_inval &= !w_inval; complete_rdma_req(req, errno, true, false); + return true; } static void rtrs_clt_recv_done(struct rtrs_clt_con *con, struct ib_wc *wc) @@ -562,7 +571,8 @@ static void rtrs_clt_rkey_rsp_done(struct rtrs_clt_con *con, struct ib_wc *wc) if (WARN_ON(buf_id != msg_id)) goto out; clt_path->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey); - process_io_rsp(clt_path, msg_id, err, w_inval); + if (!process_io_rsp(clt_path, msg_id, err, w_inval)) + goto out; } ib_dma_sync_single_for_device(clt_path->s.dev->ib_dev, iu->dma_addr, iu->size, DMA_FROM_DEVICE); @@ -577,25 +587,21 @@ static struct ib_cqe io_comp_cqe = { .done = rtrs_clt_rdma_done }; -/* - * Post x2 empty WRs: first is for this RDMA with IMM, - * second is for RECV with INV, which happened earlier. - */ -static int rtrs_post_recv_empty_x2(struct rtrs_con *con, struct ib_cqe *cqe) +static int rtrs_clt_repost_recv(struct rtrs_clt_con *con) { - struct ib_recv_wr wr_arr[2], *wr; - int i; + struct rtrs_clt_path *clt_path = to_clt_path(con->c.path); + int err; - memset(wr_arr, 0, sizeof(wr_arr)); - for (i = 0; i < ARRAY_SIZE(wr_arr); i++) { - wr = &wr_arr[i]; - wr->wr_cqe = cqe; - if (i) - /* Chain backwards */ - wr->next = &wr_arr[i - 1]; + if (READ_ONCE(clt_path->state) != RTRS_CLT_CONNECTED) + return 0; + + err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); + if (err) { + rtrs_err(con->c.path, "rtrs_post_recv_empty(): %pe\n", + ERR_PTR(err)); + rtrs_rdma_error_recovery(con); } - - return ib_post_recv(con->qp, wr, NULL); + return err; } static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) @@ -634,7 +640,13 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) w_inval = (imm_type == RTRS_IO_RSP_W_INV_IMM); rtrs_from_io_rsp_imm(imm_payload, &msg_id, &err); - process_io_rsp(clt_path, msg_id, err, w_inval); + if (!process_io_rsp(clt_path, msg_id, err, w_inval)) { + rtrs_err(clt_path->clt, + "Invalid IO rsp: msg_id %u queue_depth %zu\n", + msg_id, clt_path->queue_depth); + rtrs_rdma_error_recovery(con); + return; + } } else if (imm_type == RTRS_HB_MSG_IMM) { WARN_ON(con->c.cid); rtrs_send_hb_ack(&clt_path->s); @@ -650,19 +662,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) rtrs_wrn(con->c.path, "Unknown IMM type %u\n", imm_type); } - if (w_inval) - /* - * Post x2 empty WRs: first is for this RDMA with IMM, - * second is for RECV with INV, which happened earlier. - */ - err = rtrs_post_recv_empty_x2(&con->c, &io_comp_cqe); - else - err = rtrs_post_recv_empty(&con->c, &io_comp_cqe); - if (err) { - rtrs_err(con->c.path, "rtrs_post_recv_empty(): %pe\n", - ERR_PTR(err)); - rtrs_rdma_error_recovery(con); - } + rtrs_clt_repost_recv(con); break; case IB_WC_RECV: /* @@ -678,6 +678,7 @@ static void rtrs_clt_rdma_done(struct ib_cq *cq, struct ib_wc *wc) return rtrs_clt_rkey_rsp_done(con, wc); } + rtrs_clt_repost_recv(con); break; case IB_WC_RDMA_WRITE: /* @@ -1693,7 +1694,7 @@ static int create_con_cq_qp(struct rtrs_clt_con *con) clt_path->s.dev_ref++; /* QD * (REQ + RSP + FR REGS or INVS) + drain */ max_send_wr = min(wr_limit, clt_path->queue_depth * 4 + 1); - max_recv_wr = min(wr_limit, clt_path->queue_depth * 3 + 1); + max_recv_wr = min(wr_limit, clt_path->queue_depth * 2 + 1); max_send_sge = 2; } atomic_set(&con->c.sq_wr_avail, max_send_wr); @@ -1853,6 +1854,10 @@ static int rtrs_rdma_conn_established(struct rtrs_clt_con *con, } if (con->c.cid == 0) { queue_depth = le16_to_cpu(msg->queue_depth); + if (!queue_depth) { + rtrs_err(clt, "Invalid queue depth %u\n", queue_depth); + return -ECONNRESET; + } if (clt_path->queue_depth > 0 && queue_depth != clt_path->queue_depth) { rtrs_err(clt, "Error: queue depth changed\n"); @@ -2514,7 +2519,12 @@ static void rtrs_clt_info_rsp_done(struct ib_cq *cq, struct ib_wc *wc) ib_wc_status_msg(wc->status)); goto out; } - WARN_ON(wc->opcode != IB_WC_RECV); + if (wc->opcode != IB_WC_RECV) { + rtrs_err(clt_path->clt, + "Path info response has unexpected opcode %d\n", + wc->opcode); + goto out; + } if (wc->byte_len < sizeof(*msg)) { rtrs_err(clt_path->clt, "Path info response is malformed: size %d\n", diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 955f36efeebd..5811f02df980 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -3499,7 +3499,7 @@ static int srp_parse_options(struct net *net, const char *buf, } ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p, &has_port); - if (!has_port) + if (ret == 0 && !has_port) ret = -EINVAL; if (ret < 0) { pr_warn("bad dest parameter '%s'\n", p); diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index 7197d95f2216..43c3ff1ad8e7 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -1888,7 +1888,7 @@ static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) * We divide up our send queue size into half SEND WRs to send the * completions, and half R/W contexts to actually do the RDMA * READ/WRITE transfers. Note that we need to allocate CQ slots for - * both both, as RDMA contexts will also post completions for the + * both, as RDMA contexts will also post completions for the * RDMA READ case. */ qp_init->cap.max_send_wr = min(sq_size / 2, attrs->max_qp_wr); diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.h b/drivers/infiniband/ulp/srpt/ib_srpt.h index 4c46b301eea1..506c11b84359 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.h +++ b/drivers/infiniband/ulp/srpt/ib_srpt.h @@ -350,7 +350,7 @@ struct srpt_nexus { * struct srpt_port_attrib - attributes for SRPT port * @srp_max_rdma_size: Maximum size of SRP RDMA transfers for new connections. * @srp_max_rsp_size: Maximum size of SRP response messages in bytes. - * @srp_sq_size: Shared receive queue (SRQ) size. + * @srp_sq_size: Send queue size. * @use_srq: Whether or not to use SRQ. */ struct srpt_port_attrib { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 19e50f0d55af..a1871259210d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -18,7 +18,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \ diag/fw_tracer.o diag/crdump.o devlink.o sh_devlink.o diag/rsc_dump.o \ diag/reporter_vnic.o fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o \ - lib/nv_param.o + lib/nv_param.o data_direct.o # # Netdev basic diff --git a/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c b/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c new file mode 100644 index 000000000000..5afb11a2a9f0 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/data_direct.c @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* + * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#include +#include +#include +#include +#include +#include + +#include "mlx5_core.h" + +static LIST_HEAD(mlx5_data_direct_dev_list); +static LIST_HEAD(mlx5_data_direct_reg_list); + +/* + * This mutex should be held when accessing either of the above lists + */ +static DEFINE_MUTEX(mlx5_data_direct_mutex); + +struct mlx5_data_direct_registration { + char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; + struct list_head list; + struct blocking_notifier_head users; + struct mlx5_data_direct dd; +}; + +static int mlx5_data_direct_query_vuid(struct mlx5_core_dev *dev, + char *out_vuid) +{ + u8 out[MLX5_ST_SZ_BYTES(query_vuid_out) + + MLX5_ST_SZ_BYTES(array1024_auto)] = {}; + u8 in[MLX5_ST_SZ_BYTES(query_vuid_in)] = {}; + char *vuid; + int err; + + MLX5_SET(query_vuid_in, in, opcode, MLX5_CMD_OPCODE_QUERY_VUID); + MLX5_SET(query_vuid_in, in, vhca_id, MLX5_CAP_GEN(dev, vhca_id)); + MLX5_SET(query_vuid_in, in, data_direct, 1); + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + vuid = MLX5_ADDR_OF(query_vuid_out, out, vuid); + memcpy(out_vuid, vuid, MLX5_ST_SZ_BYTES(array1024_auto)); + return 0; +} + +static const struct pci_device_id mlx5_data_direct_pci_table[] = { + { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */ + { 0, } +}; + +static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev) +{ + struct pci_dev *pdev = dev->pdev; + unsigned int vpd_size, kw_len; + u8 *vpd_data; + int start; + int ret; + + vpd_data = pci_vpd_alloc(pdev, &vpd_size); + if (IS_ERR(vpd_data)) { + pci_err(pdev, "Unable to read VPD, err=%pe\n", vpd_data); + return PTR_ERR(vpd_data); + } + + start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "VU", &kw_len); + if (start < 0) { + ret = start; + pci_err(pdev, "VU keyword not found, err=%d\n", ret); + goto end; + } + + dev->vuid = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL); + ret = dev->vuid ? 0 : -ENOMEM; + +end: + kfree(vpd_data); + return ret; +} + +static void mlx5_data_direct_shutdown(struct pci_dev *pdev) +{ + pci_disable_device(pdev); +} + +static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) +{ + int err; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_warn(&pdev->dev, + "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err); + return err; + } + } + + dma_set_max_seg_size(&pdev->dev, SZ_2G); + return 0; +} + +static int mlx5_data_direct_create_resources(struct mlx5_core_dev *mdev) +{ + int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); + bool ro_supp = false; + void *mkc; + u32 mkey; + u32 pdn; + u32 *in; + int err; + + err = mlx5_core_alloc_pd(mdev, &pdn); + if (err) + return err; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err; + } + + MLX5_SET(create_mkey_in, in, data_direct, 1); + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); + MLX5_SET(mkc, mkc, lw, 1); + MLX5_SET(mkc, mkc, lr, 1); + MLX5_SET(mkc, mkc, rw, 1); + MLX5_SET(mkc, mkc, rr, 1); + MLX5_SET(mkc, mkc, a, 1); + MLX5_SET(mkc, mkc, pd, pdn); + MLX5_SET(mkc, mkc, length64, 1); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); + if (err) + goto err_mkey; + + mdev->data_direct->mkey = mkey; + mdev->data_direct->pdn = pdn; + + /* create another mkey with RO support */ + if (MLX5_CAP_GEN(mdev, relaxed_ordering_write)) { + MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); + ro_supp = true; + } + + if (MLX5_CAP_GEN(mdev, relaxed_ordering_read)) { + MLX5_SET(mkc, mkc, relaxed_ordering_read, 1); + ro_supp = true; + } + + if (ro_supp) { + err = mlx5_core_create_mkey(mdev, &mkey, in, inlen); + /* RO is defined as best effort */ + if (!err) { + mdev->data_direct->mkey_ro = mkey; + mdev->data_direct->mkey_ro_valid = true; + } + } + + kvfree(in); + return 0; + +err_mkey: + kvfree(in); +err: + mlx5_core_dealloc_pd(mdev, pdn); + return err; +} + +static void mlx5_data_direct_free_resources(struct mlx5_core_dev *mdev) +{ + struct mlx5_data_direct *data_direct = mdev->data_direct; + + if (data_direct->mkey_ro_valid) + mlx5_core_destroy_mkey(mdev, data_direct->mkey_ro); + + mlx5_core_destroy_mkey(mdev, data_direct->mkey); + mlx5_core_dealloc_pd(mdev, data_direct->pdn); +} + +static void mlx5_data_direct_bind(struct mlx5_data_direct_registration *reg, + struct mlx5_data_direct_dev *dev) +{ + WRITE_ONCE(reg->dd.dev, dev); +} + +static void +mlx5_data_direct_do_unbind(struct mlx5_data_direct_registration *reg) +{ + WRITE_ONCE(reg->dd.dev, NULL); + blocking_notifier_call_chain(®->users, MLX5_DATA_DIRECT_UNBIND, + NULL); +} + +int mlx5_data_direct_init(struct mlx5_core_dev *mdev) +{ + struct mlx5_data_direct_registration *reg; + struct mlx5_data_direct_dev *dev; + int err; + + if (!mlx5_data_direct_supported(mdev)) + return 0; + + reg = kzalloc_obj(*reg); + if (!reg) + return -ENOMEM; + + BLOCKING_INIT_NOTIFIER_HEAD(®->users); + + err = mlx5_data_direct_query_vuid(mdev, reg->vuid); + if (err) { + mlx5_core_warn(mdev, "Failed to query VUID, err=%d\n", err); + kfree(reg); + return -EINVAL; + } + + mdev->data_direct = ®->dd; + + err = mlx5_data_direct_create_resources(mdev); + if (err) + goto err_resources; + + mutex_lock(&mlx5_data_direct_mutex); + list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { + if (strcmp(dev->vuid, reg->vuid) == 0) { + mlx5_data_direct_bind(reg, dev); + break; + } + } + + /* Add the registration to its global list, to be used upon bind/unbind + * of its affiliated data direct device. + */ + list_add_tail(®->list, &mlx5_data_direct_reg_list); + mutex_unlock(&mlx5_data_direct_mutex); + return 0; + +err_resources: + mdev->data_direct = NULL; + kfree(reg); + return err; +} + +void mlx5_data_direct_cleanup(struct mlx5_core_dev *mdev) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(mdev)) + return; + + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); + mutex_lock(&mlx5_data_direct_mutex); + list_del(®->list); + mlx5_data_direct_do_unbind(reg); + mutex_unlock(&mlx5_data_direct_mutex); + + mlx5_data_direct_free_resources(mdev); + mdev->data_direct = NULL; + kfree(reg); +} + +int mlx5_data_direct_register(struct mlx5_core_dev *mdev, + struct notifier_block *nb) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(mdev)) + return 0; + + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_register(®->users, nb); + + return 0; +} +EXPORT_SYMBOL_GPL(mlx5_data_direct_register); + +void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev, + struct notifier_block *nb) +{ + struct mlx5_data_direct_registration *reg; + + if (!mlx5_data_direct_supported(mdev)) + return; + + reg = container_of(mdev->data_direct, + struct mlx5_data_direct_registration, dd); + blocking_notifier_chain_unregister(®->users, nb); +} +EXPORT_SYMBOL_GPL(mlx5_data_direct_unregister); + +static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) +{ + struct mlx5_data_direct_registration *reg; + + mutex_lock(&mlx5_data_direct_mutex); + list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { + if (strcmp(dev->vuid, reg->vuid) == 0) + mlx5_data_direct_bind(reg, dev); + } + + /* Add the data direct device to the global list, further mlx5 devices + * may use it later as well. + */ + list_add_tail(&dev->list, &mlx5_data_direct_dev_list); + mutex_unlock(&mlx5_data_direct_mutex); +} + +static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev) +{ + struct mlx5_data_direct_registration *reg; + + mutex_lock(&mlx5_data_direct_mutex); + /* Prevent any further affiliations */ + list_del(&dev->list); + list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { + if (strcmp(dev->vuid, reg->vuid) == 0) + mlx5_data_direct_do_unbind(reg); + } + mutex_unlock(&mlx5_data_direct_mutex); +} + +static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct mlx5_data_direct_dev *dev; + int err; + + dev = kzalloc_obj(*dev); + if (!dev) + return -ENOMEM; + + dev->device = &pdev->dev; + dev->pdev = pdev; + + pci_set_drvdata(dev->pdev, dev); + err = pci_enable_device(pdev); + if (err) { + dev_err(dev->device, "Cannot enable PCI device, err=%d\n", err); + goto err; + } + + pci_set_master(pdev); + err = mlx5_data_direct_set_dma_caps(pdev); + if (err) + goto err_disable; + + if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && + pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) && + pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) + dev_dbg(dev->device, "Enabling pci atomics failed\n"); + + err = mlx5_data_direct_vpd_get_vuid(dev); + if (err) + goto err_disable; + + mlx5_data_direct_dev_reg(dev); + return 0; + +err_disable: + pci_disable_device(pdev); +err: + kfree(dev); + return err; +} + +static void mlx5_data_direct_remove(struct pci_dev *pdev) +{ + struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev); + + mlx5_data_direct_dev_unreg(dev); + pci_disable_device(pdev); + kfree(dev->vuid); + kfree(dev); +} + +static struct pci_driver mlx5_data_direct_driver = { + .name = "mlx5_dd", + .id_table = mlx5_data_direct_pci_table, + .probe = mlx5_data_direct_probe, + .remove = mlx5_data_direct_remove, + .shutdown = mlx5_data_direct_shutdown, +}; + +int mlx5_data_direct_driver_register(void) +{ + return pci_register_driver(&mlx5_data_direct_driver); +} + +void mlx5_data_direct_driver_unregister(void) +{ + pci_unregister_driver(&mlx5_data_direct_driver); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 5f28d906c35b..453137f4bc9c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -1339,8 +1340,14 @@ static int mlx5_load(struct mlx5_core_dev *dev) if (err) goto err_traps_reg; + err = mlx5_data_direct_init(dev); + if (err) + goto err_data_direct; + return 0; +err_data_direct: + mlx5_devlink_traps_unregister(priv_to_devlink(dev)); err_traps_reg: mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); @@ -1371,6 +1378,7 @@ static int mlx5_load(struct mlx5_core_dev *dev) static void mlx5_unload(struct mlx5_core_dev *dev) { + mlx5_data_direct_cleanup(dev); mlx5_devlink_traps_unregister(priv_to_devlink(dev)); mlx5_vhca_event_stop(dev); mlx5_sf_dev_table_destroy(dev); @@ -2363,6 +2371,10 @@ static int __init mlx5_init(void) if (err) goto err_sf; + err = mlx5_data_direct_driver_register(); + if (err) + goto err_dd; + err = pci_register_driver(&mlx5_core_driver); if (err) goto err_pci; @@ -2370,6 +2382,8 @@ static int __init mlx5_init(void) return 0; err_pci: + mlx5_data_direct_driver_unregister(); +err_dd: mlx5_sf_driver_unregister(); err_sf: mlx5e_cleanup(); @@ -2381,6 +2395,7 @@ static int __init mlx5_init(void) static void __exit mlx5_cleanup(void) { pci_unregister_driver(&mlx5_core_driver); + mlx5_data_direct_driver_unregister(); mlx5_sf_driver_unregister(); mlx5e_cleanup(); mlx5_unregister_debugfs(); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index d6713a2ce676..d84d0424d04e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -532,4 +532,9 @@ static inline bool mlx5_pcie_cong_event_supported(struct mlx5_core_dev *dev) return true; } + +int mlx5_data_direct_init(struct mlx5_core_dev *mdev); +void mlx5_data_direct_cleanup(struct mlx5_core_dev *mdev); +int mlx5_data_direct_driver_register(void); +void mlx5_data_direct_driver_unregister(void); #endif /* __MLX5_CORE_H__ */ diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 8e9bfc1d6a2a..740005782fa3 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -559,11 +559,13 @@ static int mana_gd_disable_queue(struct gdma_queue *queue) static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index, enum gdma_queue_type q_type, u32 qid, - u32 tail_ptr, u8 num_req) + u32 tail_ptr, u8 num_req, u8 client_offset) { void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index; union gdma_doorbell_entry e = {}; + addr += client_offset; + switch (q_type) { case GDMA_EQ: e.eq.id = qid; @@ -623,10 +625,18 @@ void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue) * wqe_cnt for Receive Queues. This value is not used in Send Queues. */ mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, - queue->id, queue->head * GDMA_WQE_BU_SIZE, 0); + queue->id, queue->head * GDMA_WQE_BU_SIZE, 0, 0); } EXPORT_SYMBOL_NS(mana_gd_wq_ring_doorbell, "NET_MANA"); +void mana_gd_wq_ring_doorbell_ext(struct gdma_context *gc, struct gdma_queue *queue, + u32 tail_ptr, u8 wqe_cnt, u8 client_offset) +{ + mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, + queue->id, tail_ptr, wqe_cnt, client_offset); +} +EXPORT_SYMBOL_NS(mana_gd_wq_ring_doorbell_ext, "NET_MANA"); + void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) { struct gdma_context *gc = cq->gdma_dev->gdma_context; @@ -636,7 +646,7 @@ void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS); mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id, - head, arm_bit); + head, arm_bit, 0); } EXPORT_SYMBOL_NS(mana_gd_ring_cq, "NET_MANA"); @@ -653,7 +663,7 @@ void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld, dim_val |= MANA_INTR_MODR_USEC_VLD; mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, GDMA_DIM, cq->id, - dim_val, mod_comps_vld); + dim_val, mod_comps_vld, 0); } EXPORT_SYMBOL_NS(mana_gd_ring_dim, "NET_MANA"); @@ -1023,7 +1033,7 @@ static void mana_gd_process_eq_events(void *arg) head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS); mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id, - head, SET_ARM_BIT); + head, SET_ARM_BIT, 0); } static int mana_gd_register_irq(struct gdma_queue *queue, @@ -1738,6 +1748,24 @@ static void mana_gd_write_sgl(struct gdma_queue *wq, u32 sgl_offset, memcpy(mana_gd_ring_ptr(wq, sgl_offset), address, sgl_size); } +static void mana_gd_write_ib_sgl(struct gdma_queue *wq, u32 sgl_offset, + const struct gdma_wqe_request *wqe_req) +{ + const struct ib_sge *sge = wqe_req->ib_sgl; + struct gdma_sge *gdma_sgl; + u32 i; + + for (i = 0; i < wqe_req->num_sge; ++i, ++sge) { + gdma_sgl = mana_gd_ring_ptr(wq, sgl_offset); + gdma_sgl->address = sge->addr; + gdma_sgl->size = sge->length; + gdma_sgl->mem_key = sge->lkey; + sgl_offset += sizeof(*gdma_sgl); + if (sgl_offset == wq->queue_size) + sgl_offset = 0; + } +} + int mana_gd_post_work_request(struct gdma_queue *wq, const struct gdma_wqe_request *wqe_req, struct gdma_posted_wqe_info *wqe_info) @@ -1779,10 +1807,12 @@ int mana_gd_post_work_request(struct gdma_queue *wq, if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq)) return -ENOSPC; - if (wqe_info) - wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; - head = wq->head; + if (wqe_info) { + wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; + wqe_info->wqe_offset = head; + } + wqe_offset = (head * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); wqe_ptr = mana_gd_get_wqe_ptr(wq, head); oob_len = mana_gd_write_client_oob(wqe_req, wq->type, client_oob_size, @@ -1792,7 +1822,10 @@ int mana_gd_post_work_request(struct gdma_queue *wq, if (sgl_offset >= wq->queue_size) sgl_offset -= wq->queue_size; - mana_gd_write_sgl(wq, sgl_offset, wqe_req); + if (wqe_req->flags & GDMA_WR_IB_SGL) + mana_gd_write_ib_sgl(wq, sgl_offset, wqe_req); + else + mana_gd_write_sgl(wq, sgl_offset, wqe_req); wq->head += wqe_size / GDMA_WQE_BU_SIZE; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h index 15f2bc3f4043..e911fad545d6 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_if.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h @@ -7,6 +7,7 @@ #define IONIC_DEV_INFO_SIGNATURE 0x44455649 /* 'DEVI' */ #define IONIC_DEV_INFO_VERSION 1 #define IONIC_IFNAMSIZ 16 +#define IONIC_CREATE_CQ_CMD_V2_MAGIC -1 /* * enum ionic_cmd_opcode - Device commands @@ -513,6 +514,23 @@ enum ionic_lif_rdma_cap_stats { IONIC_LIF_RDMA_STAT_QP = BIT(1), }; +/** + * enum ionic_lif_rdma_alloc_qid - RDMA QID allocation capability + * + * Bitmask of RDMA queue types for which firmware can automatically + * allocate queue IDs. When a bit is set, the driver can request the + * firmware to allocate QIDs for that queue type during creation. + * + * @IONIC_LIF_RDMA_ALLOC_QID_CQ: Firmware can allocate QIDs for + * Completion Queues + * @IONIC_LIF_RDMA_ALLOC_QID_SRQ: Firmware can allocate QIDs for + * Shared Receive Queues + */ +enum ionic_lif_rdma_alloc_qid { + IONIC_LIF_RDMA_ALLOC_QID_CQ = BIT(0), + IONIC_LIF_RDMA_ALLOC_QID_SRQ = BIT(1), +}; + /** * struct ionic_lif_identity - LIF identity information (type-specific) * @@ -557,6 +575,10 @@ enum ionic_lif_rdma_cap_stats { * (enum ionic_lif_rdma_cap_stats) * @rdma.rsvd: Reserved byte * @rdma.rcq_sign_bit: RCQ sign bit + * @rdma.srq_qtype: RDMA Shared Receive Qtype + * @rdma.rsvd2: Reserved byte(s) + * @rdma.alloc_qid_cap: RDMA queue type QID allocation capability + * (bitmask of enum ionic_lif_rdma_alloc_qid) * @rdma.rsvd1: Reserved byte(s) * @words: word access to struct contents */ @@ -604,7 +626,10 @@ union ionic_lif_identity { __le16 stats_type; u8 rsvd; u8 rcq_sign_bit; - u8 rsvd1[160]; + struct ionic_lif_logical_qtype srq_qtype; + u8 rsvd2[5]; + u8 alloc_qid_cap; + u8 rsvd1[142]; } __packed rdma; } __packed; __le32 words[478]; @@ -2608,6 +2633,7 @@ struct ionic_rdma_reset_cmd { * @depth_log2: log base two of queue depth * @stride_log2: log base two of queue stride * @dma_addr: address of the queue memory + * @udma_idx: udma index * @rsvd2: reserved byte(s) * * The same command struct is used to create an RDMA event queue, completion @@ -2637,7 +2663,8 @@ struct ionic_rdma_queue_cmd { u8 depth_log2; u8 stride_log2; __le64 dma_addr; - u8 rsvd2[40]; + u8 udma_idx; + u8 rsvd2[39]; }; /****************************************************************** diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h new file mode 100644 index 000000000000..4cff5ff5489d --- /dev/null +++ b/include/linux/mlx5/data_direct.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#ifndef _MLX5_DATA_DIRECT_H +#define _MLX5_DATA_DIRECT_H + +#include +#include + +struct device; +struct mlx5_core_dev; +struct notifier_block; +struct pci_dev; + +enum mlx5_data_direct_event { + MLX5_DATA_DIRECT_UNBIND, +}; + +struct mlx5_data_direct_dev { + struct device *device; + struct pci_dev *pdev; + char *vuid; + struct list_head list; +}; + +struct mlx5_data_direct { + struct mlx5_data_direct_dev *dev; + u32 pdn; + u32 mkey; + u32 mkey_ro; + u8 mkey_ro_valid :1; +}; + +static inline struct mlx5_data_direct_dev * +mlx5_data_direct_get_dev(struct mlx5_data_direct *dd) +{ + return dd ? READ_ONCE(dd->dev) : NULL; +} + +int mlx5_data_direct_register(struct mlx5_core_dev *mdev, + struct notifier_block *nb); +void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev, + struct notifier_block *nb); + +#endif diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 83d0a83bbfbc..abb99371ff13 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -707,6 +707,7 @@ struct mlx5_rsvd_gids { struct mlx5_clock; struct mlx5_clock_dev_state; +struct mlx5_data_direct; struct mlx5_dm; struct mlx5_fw_tracer; struct mlx5_vxlan; @@ -801,6 +802,7 @@ struct mlx5_core_dev { /* sync write combining state */ struct mutex wc_state_lock; struct devlink *shd; + struct mlx5_data_direct *data_direct; }; struct mlx5_db { @@ -1395,4 +1397,10 @@ static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev) #define MLX5_SW_IMAGE_GUID_MAX_BYTES 9 +static inline bool mlx5_data_direct_supported(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, data_direct) && + MLX5_CAP_GEN_2(mdev, query_vuid); +} + #endif /* MLX5_DRIVER_H */ diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 308950f9b54b..2a28fcaa637d 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -6,6 +6,7 @@ #include #include +#include #include "shm_channel.h" @@ -54,6 +55,7 @@ enum gdma_work_request_flags { GDMA_WR_NONE = 0, GDMA_WR_OOB_IN_SGL = BIT(0), GDMA_WR_PAD_BY_SGE0 = BIT(1), + GDMA_WR_IB_SGL = BIT(2), }; enum gdma_eqe_type { @@ -212,7 +214,10 @@ struct gdma_sge { }; /* HW DATA */ struct gdma_wqe_request { - struct gdma_sge *sgl; + union { + struct gdma_sge *sgl; + struct ib_sge *ib_sgl; + }; u32 num_sge; u32 inline_oob_size; @@ -552,6 +557,7 @@ struct gdma_wqe { #define INLINE_OOB_SMALL_SIZE 8 #define INLINE_OOB_LARGE_SIZE 24 +#define INLINE_OOB_EXTRA_LARGE_SIZE 32 #define MANA_MAX_TX_WQE_SGL_ENTRIES 30 @@ -624,6 +630,8 @@ struct gdma_eqe { struct gdma_posted_wqe_info { u32 wqe_size_in_bu; + /* Unmasked WQE start offset in GDMA basic units. */ + u32 wqe_offset; }; /* GDMA_GENERATE_TEST_EQE */ @@ -1062,6 +1070,8 @@ void mana_gd_free_res_map(struct gdma_resource *r); void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue); +void mana_gd_wq_ring_doorbell_ext(struct gdma_context *gc, struct gdma_queue *queue, + u32 tail_ptr, u8 wqe_cnt, u8 client_offset); int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, struct gdma_mem_info *gmi, bool allow_scatter); diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 1fe87fd1d769..dcb645419692 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -7,6 +7,7 @@ #ifndef IB_UMEM_H #define IB_UMEM_H +#include #include struct ib_device; @@ -19,6 +20,7 @@ struct ib_umem { size_t length; unsigned long address; unsigned long dma_attrs; + enum dma_data_direction dma_dir; u32 writable : 1; u32 is_odp : 1; u32 is_dmabuf : 1; diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index cb3b6163961b..d6eaa3fee66e 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -3532,7 +3532,7 @@ static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num) * @port_num: Port number to check * * Similar to above, but specific to iWARP connections which have a different - * managment protocol than InfiniBand. + * management protocol than InfiniBand. * * Return: true if the port supports an iWARP CM (this does not guarantee that * a CM is actually running however). @@ -3577,7 +3577,7 @@ static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num) * to this group. And it should leave the group only after all queue pairs * attached to the group have been detached. * - * Return: true if the port must undertake the additional adminstrative + * Return: true if the port must undertake the additional administrative * overhead of registering/unregistering with the SM and tracking of the * total number of queue pairs attached to the multicast group. */ diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h index 80f3ba6663d0..1aaaadb8daf6 100644 --- a/include/rdma/uverbs_ioctl.h +++ b/include/rdma/uverbs_ioctl.h @@ -1142,7 +1142,7 @@ static inline int ib_is_udata_in_empty(struct ib_udata *udata) * @_udata: The system calls ib_udata struct * * Used when there is no driver response data to return. Provides forward - * compatability by zeroing any buffer the user may have provided. + * compatibility by zeroing any buffer the user may have provided. */ static inline int ib_respond_empty_udata(struct ib_udata *udata) { diff --git a/include/uapi/rdma/ionic-abi.h b/include/uapi/rdma/ionic-abi.h index 2c70ac149c4f..ac48fcd19a4f 100644 --- a/include/uapi/rdma/ionic-abi.h +++ b/include/uapi/rdma/ionic-abi.h @@ -22,6 +22,10 @@ #define IONIC_CMB_WC 8 #define IONIC_CMB_UC 16 +enum ionic_ctx_resp_comp_mask { + IONIC_CTX_CMASK_IONIC_FLAGS = 1 << 0, +}; + struct ionic_ctx_req { __u32 rsvd[2]; }; @@ -46,8 +50,9 @@ struct ionic_ctx_resp { __u8 udma_count; __u8 expdb_mask; __u8 expdb_qtypes; + __u8 rcq_sign_bit; - __u8 rsvd2[3]; + __u16 comp_mask; __aligned_u64 phc_offset; }; @@ -85,6 +90,8 @@ struct ionic_qp_req { __u8 rq_cmb; __u8 udma_mask; __u8 rsvd[3]; + __u32 ionic_flags; + __u32 rsvd_pad; }; struct ionic_qp_resp { @@ -106,7 +113,7 @@ struct ionic_srq_req { }; struct ionic_srq_resp { - __u32 qpid; + __u32 srqid; __u8 rq_cmb; __u8 udma_idx; __u8 rsvd[2]; diff --git a/include/uapi/rdma/mana-abi.h b/include/uapi/rdma/mana-abi.h index 32cbbfc80f99..ebc9a655b826 100644 --- a/include/uapi/rdma/mana-abi.h +++ b/include/uapi/rdma/mana-abi.h @@ -48,13 +48,23 @@ struct mana_ib_create_qp_resp { __u32 reserved; }; +enum mana_ib_create_rc_qp_flags { + MANA_IB_RC_QP_FIXED_WQE = 1 << 0, + MANA_IB_RC_MMQ_CREATE = 1 << 1, +}; + struct mana_ib_create_rc_qp { __aligned_u64 queue_buf[4]; __u32 queue_size[4]; + __aligned_u64 mmq_buf; + __u32 mmq_size; + __u32 comp_mask; }; struct mana_ib_create_rc_qp_resp { __u32 queue_id[4]; + __u32 mmq_id; + __u32 reserved; }; struct mana_ib_create_uc_qp { @@ -100,6 +110,8 @@ struct mana_ib_create_qp_rss_resp { enum mana_ib_ucontext_support { MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0, + MANA_IB_UCNTX_RC_EXT_SUPPORT = 1 << 1, + MANA_IB_UCNTX_RC_SQ_POW2_SUPPORT = 1 << 2, }; struct mana_ib_alloc_ucontext_resp { diff --git a/tools/testing/selftests/rdma/config b/tools/testing/selftests/rdma/config index e1ff54ec0f57..1be43c4274bb 100644 --- a/tools/testing/selftests/rdma/config +++ b/tools/testing/selftests/rdma/config @@ -1,5 +1,7 @@ -CONFIG_TUN -CONFIG_VETH -CONFIG_DUMMY -CONFIG_NET_NS -CONFIG_RDMA_RXE +CONFIG_TUN=m +CONFIG_VETH=m +CONFIG_DUMMY=m +CONFIG_NET_NS=y +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_RDMA_RXE=m diff --git a/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh b/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh index 0e4fbfeebd22..89218ccfba4c 100755 --- a/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh +++ b/tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh @@ -4,6 +4,8 @@ PORT=4791 MODS=("tun" "rdma_rxe") +source "$(dirname "$0")/../kselftest/ktap_helpers.sh" + exec > /dev/null # --- Helper: Cleanup Routine --- @@ -24,6 +26,10 @@ if [[ $EUID -ne 0 ]]; then fi for m in "${MODS[@]}"; do + if ! modinfo "$m" >/dev/null 2>&1; then + echo "SKIP: Kernel module '$m' not found." >&2 + exit $KSFT_SKIP + fi modprobe "$m" || { echo "Error: Failed to load $m"; exit 1; } done