diff --git a/Documentation/networking/device_drivers/ethernet/index.rst b/Documentation/networking/device_drivers/ethernet/index.rst index 786a23c84b90..d9980c84487a 100644 --- a/Documentation/networking/device_drivers/ethernet/index.rst +++ b/Documentation/networking/device_drivers/ethernet/index.rst @@ -35,6 +35,7 @@ Contents: intel/idpf intel/igb intel/igbvf + intel/ixd intel/ixgbe intel/ixgbevf intel/i40e diff --git a/Documentation/networking/device_drivers/ethernet/intel/ixd.rst b/Documentation/networking/device_drivers/ethernet/intel/ixd.rst new file mode 100644 index 000000000000..1387626e5d20 --- /dev/null +++ b/Documentation/networking/device_drivers/ethernet/intel/ixd.rst @@ -0,0 +1,39 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +========================================================================== +iXD Linux* Base Driver for the Intel(R) Control Plane Function +========================================================================== + +Intel iXD Linux driver. +Copyright(C) 2025 Intel Corporation. + +.. contents:: + +For questions related to hardware requirements, refer to the documentation +supplied with your Intel adapter. All hardware requirements listed apply to use +with Linux. + + +Identifying Your Adapter +======================== +For information on how to identify your adapter, and for the latest Intel +network drivers, refer to the Intel Support website: +http://www.intel.com/support + + +Support +======= +For general information, go to the Intel support website at: +http://www.intel.com/support/ + +If an issue is identified with the released source code on a supported kernel +with a supported adapter, email the specific information related to the issue +to intel-wired-lan@lists.osuosl.org. + + +Trademarks +========== +Intel is a trademark or registered trademark of Intel Corporation or its +subsidiaries in the United States and/or other countries. + +* Other names and brands may be claimed as the property of others. diff --git a/Documentation/networking/devlink/index.rst b/Documentation/networking/devlink/index.rst index 4745148fecf4..d4a83fdcff7f 100644 --- a/Documentation/networking/devlink/index.rst +++ b/Documentation/networking/devlink/index.rst @@ -87,6 +87,7 @@ parameters, info versions, and other features it supports. ice ionic iosm + ixd ixgbe kvaser_pciefd kvaser_usb diff --git a/Documentation/networking/devlink/ixd.rst b/Documentation/networking/devlink/ixd.rst new file mode 100644 index 000000000000..cba4f6f65013 --- /dev/null +++ b/Documentation/networking/devlink/ixd.rst @@ -0,0 +1,30 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================== +ixd devlink support +=================== + +This document describes the devlink features implemented by the ``ixd`` +device driver. + +Info versions +============= + +The ``ixd`` driver reports the following versions + +.. list-table:: devlink info versions implemented + :widths: 5 5 5 90 + + * - Name + - Type + - Example + - Description + * - ``device.type`` + - fixed + - MEV + - The hardware type for this device + * - ``fw.mgmt.api`` + - running + - 2.0 + - 2-digit version number (major.minor) of the communication channel + (virtchnl) used by the device. diff --git a/MAINTAINERS b/MAINTAINERS index b050c72e2d49..94a468d66abf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -13034,8 +13034,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/tnguy/next-queue.git F: Documentation/networking/device_drivers/ethernet/intel/ F: drivers/net/ethernet/intel/ F: drivers/net/ethernet/intel/*/ -F: include/linux/avf/virtchnl.h -F: include/linux/net/intel/*/ +F: include/linux/net/intel/ INTEL ETHERNET PROTOCOL DRIVER FOR RDMA M: Tatyana Nikolova diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index 288fa8ce53af..780f113986ea 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -398,4 +398,6 @@ config IGC_LEDS source "drivers/net/ethernet/intel/idpf/Kconfig" +source "drivers/net/ethernet/intel/ixd/Kconfig" + endif # NET_VENDOR_INTEL diff --git a/drivers/net/ethernet/intel/Makefile b/drivers/net/ethernet/intel/Makefile index 9a37dc76aef0..08b29f3b6801 100644 --- a/drivers/net/ethernet/intel/Makefile +++ b/drivers/net/ethernet/intel/Makefile @@ -19,3 +19,4 @@ obj-$(CONFIG_IAVF) += iavf/ obj-$(CONFIG_FM10K) += fm10k/ obj-$(CONFIG_ICE) += ice/ obj-$(CONFIG_IDPF) += idpf/ +obj-$(CONFIG_IXD) += ixd/ diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 83e780919ac9..1b6a8fbaa648 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -8,8 +8,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c index 59f5c1e810eb..8dadfef2c09f 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_common.c +++ b/drivers/net/ethernet/intel/i40e/i40e_common.c @@ -1,10 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2021 Intel Corporation. */ -#include #include #include #include +#include #include #include "i40e_adminq_cmd.h" #include "i40e_devids.h" diff --git a/drivers/net/ethernet/intel/i40e/i40e_prototype.h b/drivers/net/ethernet/intel/i40e/i40e_prototype.h index 26bb7bffe361..e3d57550090e 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_prototype.h +++ b/drivers/net/ethernet/intel/i40e/i40e_prototype.h @@ -5,7 +5,7 @@ #define _I40E_PROTOTYPE_H_ #include -#include +#include #include "i40e_debug.h" #include "i40e_type.h" diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h index f558b45725c8..4e119c0502f3 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h @@ -4,7 +4,7 @@ #ifndef _I40E_VIRTCHNL_PF_H_ #define _I40E_VIRTCHNL_PF_H_ -#include +#include #include #include "i40e_type.h" diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index 050f8241ef5e..dc31202b2a94 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -37,7 +38,6 @@ #include #include "iavf_type.h" -#include #include "iavf_txrx.h" #include "iavf_fdir.h" #include "iavf_adv_rss.h" diff --git a/drivers/net/ethernet/intel/iavf/iavf_common.c b/drivers/net/ethernet/intel/iavf/iavf_common.c index 614a886bca99..277193a97d91 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_common.c +++ b/drivers/net/ethernet/intel/iavf/iavf_common.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright(c) 2013 - 2018 Intel Corporation. */ -#include +#include #include #include "iavf_type.h" #include "iavf_adminq.h" diff --git a/drivers/net/ethernet/intel/iavf/iavf_prototype.h b/drivers/net/ethernet/intel/iavf/iavf_prototype.h index 7f9f9dbf959a..1b1f6ede3920 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_prototype.h +++ b/drivers/net/ethernet/intel/iavf/iavf_prototype.h @@ -4,9 +4,10 @@ #ifndef _IAVF_PROTOTYPE_H_ #define _IAVF_PROTOTYPE_H_ +#include + #include "iavf_type.h" #include "iavf_alloc.h" -#include /* Prototypes for shared code functions that are not in * the standard function pointer structures. These are diff --git a/drivers/net/ethernet/intel/iavf/iavf_types.h b/drivers/net/ethernet/intel/iavf/iavf_types.h index a095855122bf..35d6d8fcca04 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_types.h +++ b/drivers/net/ethernet/intel/iavf/iavf_types.h @@ -4,9 +4,7 @@ #ifndef _IAVF_TYPES_H_ #define _IAVF_TYPES_H_ -#include "iavf_types.h" - -#include +#include #include /* structure used to queue PTP commands for processing */ diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index fc91b6665f90..db3c7015c56c 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -36,7 +36,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 9f5344212195..d1d674ca644f 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -5,13 +5,13 @@ #define _ICE_COMMON_H_ #include +#include #include "ice.h" #include "ice_type.h" #include "ice_nvm.h" #include "ice_flex_pipe.h" #include "ice_parser.h" -#include #include "ice_switch.h" #include "ice_fdir.h" diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.h b/drivers/net/ethernet/intel/ice/ice_vf_lib.h index 7a9c75d1d07c..fa436b3b1eac 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.h @@ -8,9 +8,9 @@ #include #include #include +#include #include #include -#include #include "ice_type.h" #include "ice_flow.h" #include "virt/fdir.h" diff --git a/drivers/net/ethernet/intel/ice/virt/virtchnl.h b/drivers/net/ethernet/intel/ice/virt/virtchnl.h index 71bb456e2d71..d11789b3ae1f 100644 --- a/drivers/net/ethernet/intel/ice/virt/virtchnl.h +++ b/drivers/net/ethernet/intel/ice/virt/virtchnl.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include "ice_vf_lib.h" /* Restrict number of MAC Addr and VLAN that non-trusted VF can programmed */ diff --git a/drivers/net/ethernet/intel/idpf/Kconfig b/drivers/net/ethernet/intel/idpf/Kconfig index adab2154125b..586df3a4afe9 100644 --- a/drivers/net/ethernet/intel/idpf/Kconfig +++ b/drivers/net/ethernet/intel/idpf/Kconfig @@ -6,6 +6,7 @@ config IDPF depends on PCI_MSI depends on PTP_1588_CLOCK_OPTIONAL select DIMLIB + select LIBIE_CP select LIBETH_XDP help This driver supports Intel(R) Infrastructure Data Path Function diff --git a/drivers/net/ethernet/intel/idpf/Makefile b/drivers/net/ethernet/intel/idpf/Makefile index 651ddee942bd..4aaafa175ec3 100644 --- a/drivers/net/ethernet/intel/idpf/Makefile +++ b/drivers/net/ethernet/intel/idpf/Makefile @@ -6,8 +6,6 @@ obj-$(CONFIG_IDPF) += idpf.o idpf-y := \ - idpf_controlq.o \ - idpf_controlq_setup.o \ idpf_dev.o \ idpf_ethtool.o \ idpf_idc.o \ diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index ec1b75f039bb..470bc23c844c 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -23,10 +23,10 @@ struct idpf_rss_data; #include #include +#include +#include -#include "virtchnl2.h" #include "idpf_txrx.h" -#include "idpf_controlq.h" #define GETMAXVAL(num_bits) GENMASK((num_bits) - 1, 0) @@ -36,11 +36,10 @@ struct idpf_rss_data; #define IDPF_NUM_FILTERS_PER_MSG 20 #define IDPF_NUM_DFLT_MBX_Q 2 /* includes both TX and RX */ #define IDPF_DFLT_MBX_Q_LEN 64 -#define IDPF_DFLT_MBX_ID -1 /* maximum number of times to try before resetting mailbox */ #define IDPF_MB_MAX_ERR 20 #define IDPF_NUM_CHUNKS_PER_MSG(struct_sz, chunk_sz) \ - ((IDPF_CTLQ_MAX_BUF_LEN - (struct_sz)) / (chunk_sz)) + ((LIBIE_CTLQ_MAX_BUF_LEN - (struct_sz)) / (chunk_sz)) #define IDPF_WAIT_FOR_MARKER_TIMEO 500 #define IDPF_MAX_WAIT 500 @@ -201,8 +200,8 @@ struct idpf_vport_max_q { * @ptp_reg_init: PTP register initialization */ struct idpf_reg_ops { - void (*ctlq_reg_init)(struct idpf_adapter *adapter, - struct idpf_ctlq_create_info *cq); + void (*ctlq_reg_init)(struct libie_mmio_info *mmio, + struct libie_ctlq_create_info *cctlq_info); int (*intr_reg_init)(struct idpf_vport *vport, struct idpf_q_vec_rsrc *rsrc); void (*mb_intr_reg_init)(struct idpf_adapter *adapter); @@ -605,8 +604,6 @@ struct idpf_vport_config { DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS); }; -struct idpf_vc_xn_manager; - #define idpf_for_each_vport(adapter, iter) \ for (struct idpf_vport **__##iter = &(adapter)->vports[0], \ *iter = (adapter)->max_vports ? *__##iter : NULL; \ @@ -624,7 +621,10 @@ struct idpf_vc_xn_manager; * @state: Init state machine * @flags: See enum idpf_flags * @reset_reg: See struct idpf_reset_reg - * @hw: Device access data + * @ctlq_ctx: controlq context + * @asq: Send control queue info + * @arq: Receive control queue info + * @xnm: Xn transaction manager * @num_avail_msix: Available number of MSIX vectors * @num_msix_entries: Number of entries in MSIX table * @msix_entries: MSIX table @@ -638,7 +638,6 @@ struct idpf_vc_xn_manager; * @avail_queues: Device given queue limits * @vports: Array to store vports created by the driver * @netdevs: Associated Vport netdevs - * @vport_params_reqd: Vport params requested * @vport_params_recvd: Vport params received * @vport_ids: Array of device given vport identifiers * @singleq_pt_lkup: Lookup table for singleq RX ptypes @@ -658,7 +657,6 @@ struct idpf_vc_xn_manager; * @stats_task: Periodic statistics retrieval task * @stats_wq: Workqueue for statistics task * @caps: Negotiated capabilities with device - * @vcxn_mngr: Virtchnl transaction manager * @dev_ops: See idpf_dev_ops * @cdev_info: IDC core device info pointer * @num_vfs: Number of allocated VFs through sysfs. PF does not directly talk @@ -682,7 +680,10 @@ struct idpf_adapter { enum idpf_state state; DECLARE_BITMAP(flags, IDPF_FLAGS_NBITS); struct idpf_reset_reg reset_reg; - struct idpf_hw hw; + struct libie_ctlq_ctx ctlq_ctx; + struct libie_ctlq_info *asq; + struct libie_ctlq_info *arq; + struct libie_ctlq_xn_manager *xnm; u16 num_avail_msix; u16 num_msix_entries; struct msix_entry *msix_entries; @@ -697,7 +698,6 @@ struct idpf_adapter { struct idpf_avail_queue_info avail_queues; struct idpf_vport **vports; struct net_device **netdevs; - struct virtchnl2_create_vport **vport_params_reqd; struct virtchnl2_create_vport **vport_params_recvd; u32 *vport_ids; @@ -720,7 +720,6 @@ struct idpf_adapter { struct delayed_work stats_task; struct workqueue_struct *stats_wq; struct virtchnl2_get_capabilities caps; - struct idpf_vc_xn_manager *vcxn_mngr; struct idpf_dev_ops dev_ops; struct iidc_rdma_core_dev_info *cdev_info; @@ -872,70 +871,6 @@ static inline u8 idpf_get_min_tx_pkt_len(struct idpf_adapter *adapter) return pkt_len ? pkt_len : IDPF_TX_MIN_PKT_LEN; } -/** - * idpf_get_mbx_reg_addr - Get BAR0 mailbox register address - * @adapter: private data struct - * @reg_offset: register offset value - * - * Return: BAR0 mailbox register address based on register offset. - */ -static inline void __iomem *idpf_get_mbx_reg_addr(struct idpf_adapter *adapter, - resource_size_t reg_offset) -{ - return adapter->hw.mbx.vaddr + reg_offset; -} - -/** - * idpf_get_rstat_reg_addr - Get BAR0 rstat register address - * @adapter: private data struct - * @reg_offset: register offset value - * - * Return: BAR0 rstat register address based on register offset. - */ -static inline void __iomem *idpf_get_rstat_reg_addr(struct idpf_adapter *adapter, - resource_size_t reg_offset) -{ - reg_offset -= adapter->dev_ops.static_reg_info[1].start; - - return adapter->hw.rstat.vaddr + reg_offset; -} - -/** - * idpf_get_reg_addr - Get BAR0 register address - * @adapter: private data struct - * @reg_offset: register offset value - * - * Based on the register offset, return the actual BAR0 register address - */ -static inline void __iomem *idpf_get_reg_addr(struct idpf_adapter *adapter, - resource_size_t reg_offset) -{ - struct idpf_hw *hw = &adapter->hw; - - for (int i = 0; i < hw->num_lan_regs; i++) { - struct idpf_mmio_reg *region = &hw->lan_regs[i]; - - if (reg_offset >= region->addr_start && - reg_offset < (region->addr_start + region->addr_len)) { - /* Convert the offset so that it is relative to the - * start of the region. Then add the base address of - * the region to get the final address. - */ - reg_offset -= region->addr_start; - - return region->vaddr + reg_offset; - } - } - - /* It's impossible to hit this case with offsets from the CP. But if we - * do for any other reason, the kernel will panic on that register - * access. Might as well do it here to make it clear what's happening. - */ - BUG(); - - return NULL; -} - /** * idpf_is_reset_detected - check if we were reset at some point * @adapter: driver specific private structure @@ -944,11 +879,12 @@ static inline void __iomem *idpf_get_reg_addr(struct idpf_adapter *adapter, */ static inline bool idpf_is_reset_detected(struct idpf_adapter *adapter) { - if (!adapter->hw.arq) + struct libie_ctlq_info *arq = adapter->arq; + + if (!arq) return true; - return !(readl(idpf_get_mbx_reg_addr(adapter, adapter->hw.arq->reg.len)) & - adapter->hw.arq->reg.len_mask); + return !(readl(arq->reg.len) & arq->reg.len_mask); } /** @@ -1048,6 +984,7 @@ void idpf_vc_event_task(struct work_struct *work); void idpf_dev_ops_init(struct idpf_adapter *adapter); void idpf_vf_dev_ops_init(struct idpf_adapter *adapter); int idpf_intr_req(struct idpf_adapter *adapter); +void idpf_mb_intr_rel_irq(struct idpf_adapter *adapter); void idpf_intr_rel(struct idpf_adapter *adapter); u16 idpf_get_max_tx_hdr_size(struct idpf_adapter *adapter); int idpf_initiate_soft_reset(struct idpf_vport *vport, diff --git a/drivers/net/ethernet/intel/idpf/idpf_controlq.c b/drivers/net/ethernet/intel/idpf/idpf_controlq.c deleted file mode 100644 index d2dde43269e9..000000000000 --- a/drivers/net/ethernet/intel/idpf/idpf_controlq.c +++ /dev/null @@ -1,621 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright (C) 2023 Intel Corporation */ - -#include "idpf_controlq.h" - -/** - * idpf_ctlq_setup_regs - initialize control queue registers - * @cq: pointer to the specific control queue - * @q_create_info: structs containing info for each queue to be initialized - */ -static void idpf_ctlq_setup_regs(struct idpf_ctlq_info *cq, - struct idpf_ctlq_create_info *q_create_info) -{ - /* set control queue registers in our local struct */ - cq->reg.head = q_create_info->reg.head; - cq->reg.tail = q_create_info->reg.tail; - cq->reg.len = q_create_info->reg.len; - cq->reg.bah = q_create_info->reg.bah; - cq->reg.bal = q_create_info->reg.bal; - cq->reg.len_mask = q_create_info->reg.len_mask; - cq->reg.len_ena_mask = q_create_info->reg.len_ena_mask; - cq->reg.head_mask = q_create_info->reg.head_mask; -} - -/** - * idpf_ctlq_init_regs - Initialize control queue registers - * @hw: pointer to hw struct - * @cq: pointer to the specific Control queue - * @is_rxq: true if receive control queue, false otherwise - * - * Initialize registers. The caller is expected to have already initialized the - * descriptor ring memory and buffer memory - */ -static void idpf_ctlq_init_regs(struct idpf_hw *hw, struct idpf_ctlq_info *cq, - bool is_rxq) -{ - /* Update tail to post pre-allocated buffers for rx queues */ - if (is_rxq) - idpf_mbx_wr32(hw, cq->reg.tail, (u32)(cq->ring_size - 1)); - - /* For non-Mailbox control queues only TAIL need to be set */ - if (cq->q_id != -1) - return; - - /* Clear Head for both send or receive */ - idpf_mbx_wr32(hw, cq->reg.head, 0); - - /* set starting point */ - idpf_mbx_wr32(hw, cq->reg.bal, lower_32_bits(cq->desc_ring.pa)); - idpf_mbx_wr32(hw, cq->reg.bah, upper_32_bits(cq->desc_ring.pa)); - idpf_mbx_wr32(hw, cq->reg.len, (cq->ring_size | cq->reg.len_ena_mask)); -} - -/** - * idpf_ctlq_init_rxq_bufs - populate receive queue descriptors with buf - * @cq: pointer to the specific Control queue - * - * Record the address of the receive queue DMA buffers in the descriptors. - * The buffers must have been previously allocated. - */ -static void idpf_ctlq_init_rxq_bufs(struct idpf_ctlq_info *cq) -{ - int i; - - for (i = 0; i < cq->ring_size; i++) { - struct idpf_ctlq_desc *desc = IDPF_CTLQ_DESC(cq, i); - struct idpf_dma_mem *bi = cq->bi.rx_buff[i]; - - /* No buffer to post to descriptor, continue */ - if (!bi) - continue; - - desc->flags = - cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD); - desc->opcode = 0; - desc->datalen = cpu_to_le16(bi->size); - desc->ret_val = 0; - desc->v_opcode_dtype = 0; - desc->v_retval = 0; - desc->params.indirect.addr_high = - cpu_to_le32(upper_32_bits(bi->pa)); - desc->params.indirect.addr_low = - cpu_to_le32(lower_32_bits(bi->pa)); - desc->params.indirect.param0 = 0; - desc->params.indirect.sw_cookie = 0; - desc->params.indirect.v_flags = 0; - } -} - -/** - * idpf_ctlq_shutdown - shutdown the CQ - * @hw: pointer to hw struct - * @cq: pointer to the specific Control queue - * - * The main shutdown routine for any controq queue - */ -static void idpf_ctlq_shutdown(struct idpf_hw *hw, struct idpf_ctlq_info *cq) -{ - spin_lock(&cq->cq_lock); - - /* free ring buffers and the ring itself */ - idpf_ctlq_dealloc_ring_res(hw, cq); - - /* Set ring_size to 0 to indicate uninitialized queue */ - cq->ring_size = 0; - - spin_unlock(&cq->cq_lock); -} - -/** - * idpf_ctlq_add - add one control queue - * @hw: pointer to hardware struct - * @qinfo: info for queue to be created - * @cq_out: (output) double pointer to control queue to be created - * - * Allocate and initialize a control queue and add it to the control queue list. - * The cq parameter will be allocated/initialized and passed back to the caller - * if no errors occur. - * - * Note: idpf_ctlq_init must be called prior to any calls to idpf_ctlq_add - */ -int idpf_ctlq_add(struct idpf_hw *hw, - struct idpf_ctlq_create_info *qinfo, - struct idpf_ctlq_info **cq_out) -{ - struct idpf_ctlq_info *cq; - bool is_rxq = false; - int err; - - cq = kzalloc_obj(*cq); - if (!cq) - return -ENOMEM; - - cq->cq_type = qinfo->type; - cq->q_id = qinfo->id; - cq->buf_size = qinfo->buf_size; - cq->ring_size = qinfo->len; - - cq->next_to_use = 0; - cq->next_to_clean = 0; - cq->next_to_post = cq->ring_size - 1; - - switch (qinfo->type) { - case IDPF_CTLQ_TYPE_MAILBOX_RX: - is_rxq = true; - fallthrough; - case IDPF_CTLQ_TYPE_MAILBOX_TX: - err = idpf_ctlq_alloc_ring_res(hw, cq); - break; - default: - err = -EBADR; - break; - } - - if (err) - goto init_free_q; - - if (is_rxq) { - idpf_ctlq_init_rxq_bufs(cq); - } else { - /* Allocate the array of msg pointers for TX queues */ - cq->bi.tx_msg = kzalloc_objs(struct idpf_ctlq_msg *, qinfo->len); - if (!cq->bi.tx_msg) { - err = -ENOMEM; - goto init_dealloc_q_mem; - } - } - - idpf_ctlq_setup_regs(cq, qinfo); - - idpf_ctlq_init_regs(hw, cq, is_rxq); - - spin_lock_init(&cq->cq_lock); - - list_add(&cq->cq_list, &hw->cq_list_head); - - *cq_out = cq; - - return 0; - -init_dealloc_q_mem: - /* free ring buffers and the ring itself */ - idpf_ctlq_dealloc_ring_res(hw, cq); -init_free_q: - kfree(cq); - - return err; -} - -/** - * idpf_ctlq_remove - deallocate and remove specified control queue - * @hw: pointer to hardware struct - * @cq: pointer to control queue to be removed - */ -void idpf_ctlq_remove(struct idpf_hw *hw, - struct idpf_ctlq_info *cq) -{ - list_del(&cq->cq_list); - idpf_ctlq_shutdown(hw, cq); - kfree(cq); -} - -/** - * idpf_ctlq_init - main initialization routine for all control queues - * @hw: pointer to hardware struct - * @num_q: number of queues to initialize - * @q_info: array of structs containing info for each queue to be initialized - * - * This initializes any number and any type of control queues. This is an all - * or nothing routine; if one fails, all previously allocated queues will be - * destroyed. This must be called prior to using the individual add/remove - * APIs. - */ -int idpf_ctlq_init(struct idpf_hw *hw, u8 num_q, - struct idpf_ctlq_create_info *q_info) -{ - struct idpf_ctlq_info *cq, *tmp; - int err; - int i; - - INIT_LIST_HEAD(&hw->cq_list_head); - - for (i = 0; i < num_q; i++) { - struct idpf_ctlq_create_info *qinfo = q_info + i; - - err = idpf_ctlq_add(hw, qinfo, &cq); - if (err) - goto init_destroy_qs; - } - - return 0; - -init_destroy_qs: - list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list) - idpf_ctlq_remove(hw, cq); - - return err; -} - -/** - * idpf_ctlq_deinit - destroy all control queues - * @hw: pointer to hw struct - */ -void idpf_ctlq_deinit(struct idpf_hw *hw) -{ - struct idpf_ctlq_info *cq, *tmp; - - list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list) - idpf_ctlq_remove(hw, cq); -} - -/** - * idpf_ctlq_send - send command to Control Queue (CTQ) - * @hw: pointer to hw struct - * @cq: handle to control queue struct to send on - * @num_q_msg: number of messages to send on control queue - * @q_msg: pointer to array of queue messages to be sent - * - * The caller is expected to allocate DMAable buffers and pass them to the - * send routine via the q_msg struct / control queue specific data struct. - * The control queue will hold a reference to each send message until - * the completion for that message has been cleaned. - */ -int idpf_ctlq_send(struct idpf_hw *hw, struct idpf_ctlq_info *cq, - u16 num_q_msg, struct idpf_ctlq_msg q_msg[]) -{ - struct idpf_ctlq_desc *desc; - int num_desc_avail; - int err = 0; - int i; - - spin_lock(&cq->cq_lock); - - /* Ensure there are enough descriptors to send all messages */ - num_desc_avail = IDPF_CTLQ_DESC_UNUSED(cq); - if (num_desc_avail == 0 || num_desc_avail < num_q_msg) { - err = -ENOSPC; - goto err_unlock; - } - - for (i = 0; i < num_q_msg; i++) { - struct idpf_ctlq_msg *msg = &q_msg[i]; - - desc = IDPF_CTLQ_DESC(cq, cq->next_to_use); - - desc->opcode = cpu_to_le16(msg->opcode); - desc->pfid_vfid = cpu_to_le16(msg->func_id); - - desc->v_opcode_dtype = cpu_to_le32(msg->cookie.mbx.chnl_opcode); - desc->v_retval = cpu_to_le32(msg->cookie.mbx.chnl_retval); - - desc->flags = cpu_to_le16((msg->host_id & IDPF_HOST_ID_MASK) << - IDPF_CTLQ_FLAG_HOST_ID_S); - if (msg->data_len) { - struct idpf_dma_mem *buff = msg->ctx.indirect.payload; - - desc->datalen |= cpu_to_le16(msg->data_len); - desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_BUF); - desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_RD); - - /* Update the address values in the desc with the pa - * value for respective buffer - */ - desc->params.indirect.addr_high = - cpu_to_le32(upper_32_bits(buff->pa)); - desc->params.indirect.addr_low = - cpu_to_le32(lower_32_bits(buff->pa)); - - memcpy(&desc->params, msg->ctx.indirect.context, - IDPF_INDIRECT_CTX_SIZE); - } else { - memcpy(&desc->params, msg->ctx.direct, - IDPF_DIRECT_CTX_SIZE); - } - - /* Store buffer info */ - cq->bi.tx_msg[cq->next_to_use] = msg; - - (cq->next_to_use)++; - if (cq->next_to_use == cq->ring_size) - cq->next_to_use = 0; - } - - /* Force memory write to complete before letting hardware - * know that there are new descriptors to fetch. - */ - dma_wmb(); - - idpf_mbx_wr32(hw, cq->reg.tail, cq->next_to_use); - -err_unlock: - spin_unlock(&cq->cq_lock); - - return err; -} - -/** - * idpf_ctlq_clean_sq - reclaim send descriptors on HW write back for the - * requested queue - * @cq: pointer to the specific Control queue - * @clean_count: (input|output) number of descriptors to clean as input, and - * number of descriptors actually cleaned as output - * @msg_status: (output) pointer to msg pointer array to be populated; needs - * to be allocated by caller - * - * Returns an array of message pointers associated with the cleaned - * descriptors. The pointers are to the original ctlq_msgs sent on the cleaned - * descriptors. The status will be returned for each; any messages that failed - * to send will have a non-zero status. The caller is expected to free original - * ctlq_msgs and free or reuse the DMA buffers. - */ -int idpf_ctlq_clean_sq(struct idpf_ctlq_info *cq, u16 *clean_count, - struct idpf_ctlq_msg *msg_status[]) -{ - struct idpf_ctlq_desc *desc; - u16 i, num_to_clean; - u16 ntc, desc_err; - - if (*clean_count == 0) - return 0; - if (*clean_count > cq->ring_size) - return -EBADR; - - spin_lock(&cq->cq_lock); - - ntc = cq->next_to_clean; - - num_to_clean = *clean_count; - - for (i = 0; i < num_to_clean; i++) { - /* Fetch next descriptor and check if marked as done */ - desc = IDPF_CTLQ_DESC(cq, ntc); - if (!(le16_to_cpu(desc->flags) & IDPF_CTLQ_FLAG_DD)) - break; - - /* Ensure no other fields are read until DD flag is checked */ - dma_rmb(); - - /* strip off FW internal code */ - desc_err = le16_to_cpu(desc->ret_val) & 0xff; - - msg_status[i] = cq->bi.tx_msg[ntc]; - msg_status[i]->status = desc_err; - - cq->bi.tx_msg[ntc] = NULL; - - /* Zero out any stale data */ - memset(desc, 0, sizeof(*desc)); - - ntc++; - if (ntc == cq->ring_size) - ntc = 0; - } - - cq->next_to_clean = ntc; - - spin_unlock(&cq->cq_lock); - - /* Return number of descriptors actually cleaned */ - *clean_count = i; - - return 0; -} - -/** - * idpf_ctlq_post_rx_buffs - post buffers to descriptor ring - * @hw: pointer to hw struct - * @cq: pointer to control queue handle - * @buff_count: (input|output) input is number of buffers caller is trying to - * return; output is number of buffers that were not posted - * @buffs: array of pointers to dma mem structs to be given to hardware - * - * Caller uses this function to return DMA buffers to the descriptor ring after - * consuming them; buff_count will be the number of buffers. - * - * Note: this function needs to be called after a receive call even - * if there are no DMA buffers to be returned, i.e. buff_count = 0, - * buffs = NULL to support direct commands - */ -int idpf_ctlq_post_rx_buffs(struct idpf_hw *hw, struct idpf_ctlq_info *cq, - u16 *buff_count, struct idpf_dma_mem **buffs) -{ - struct idpf_ctlq_desc *desc; - u16 ntp = cq->next_to_post; - bool buffs_avail = false; - u16 tbp = ntp + 1; - int i = 0; - - if (*buff_count > cq->ring_size) - return -EBADR; - - if (*buff_count > 0) - buffs_avail = true; - - spin_lock(&cq->cq_lock); - - if (tbp >= cq->ring_size) - tbp = 0; - - if (tbp == cq->next_to_clean) - /* Nothing to do */ - goto post_buffs_out; - - /* Post buffers for as many as provided or up until the last one used */ - while (ntp != cq->next_to_clean) { - desc = IDPF_CTLQ_DESC(cq, ntp); - - if (cq->bi.rx_buff[ntp]) - goto fill_desc; - if (!buffs_avail) { - /* If the caller hasn't given us any buffers or - * there are none left, search the ring itself - * for an available buffer to move to this - * entry starting at the next entry in the ring - */ - tbp = ntp + 1; - - /* Wrap ring if necessary */ - if (tbp >= cq->ring_size) - tbp = 0; - - while (tbp != cq->next_to_clean) { - if (cq->bi.rx_buff[tbp]) { - cq->bi.rx_buff[ntp] = - cq->bi.rx_buff[tbp]; - cq->bi.rx_buff[tbp] = NULL; - - /* Found a buffer, no need to - * search anymore - */ - break; - } - - /* Wrap ring if necessary */ - tbp++; - if (tbp >= cq->ring_size) - tbp = 0; - } - - if (tbp == cq->next_to_clean) - goto post_buffs_out; - } else { - /* Give back pointer to DMA buffer */ - cq->bi.rx_buff[ntp] = buffs[i]; - i++; - - if (i >= *buff_count) - buffs_avail = false; - } - -fill_desc: - desc->flags = - cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD); - - /* Post buffers to descriptor */ - desc->datalen = cpu_to_le16(cq->bi.rx_buff[ntp]->size); - desc->params.indirect.addr_high = - cpu_to_le32(upper_32_bits(cq->bi.rx_buff[ntp]->pa)); - desc->params.indirect.addr_low = - cpu_to_le32(lower_32_bits(cq->bi.rx_buff[ntp]->pa)); - - ntp++; - if (ntp == cq->ring_size) - ntp = 0; - } - -post_buffs_out: - /* Only update tail if buffers were actually posted */ - if (cq->next_to_post != ntp) { - if (ntp) - /* Update next_to_post to ntp - 1 since current ntp - * will not have a buffer - */ - cq->next_to_post = ntp - 1; - else - /* Wrap to end of end ring since current ntp is 0 */ - cq->next_to_post = cq->ring_size - 1; - - dma_wmb(); - - idpf_mbx_wr32(hw, cq->reg.tail, cq->next_to_post); - } - - spin_unlock(&cq->cq_lock); - - /* return the number of buffers that were not posted */ - *buff_count = *buff_count - i; - - return 0; -} - -/** - * idpf_ctlq_recv - receive control queue message call back - * @cq: pointer to control queue handle to receive on - * @num_q_msg: (input|output) input number of messages that should be received; - * output number of messages actually received - * @q_msg: (output) array of received control queue messages on this q; - * needs to be pre-allocated by caller for as many messages as requested - * - * Called by interrupt handler or polling mechanism. Caller is expected - * to free buffers - */ -int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg, - struct idpf_ctlq_msg *q_msg) -{ - u16 num_to_clean, ntc, flags; - struct idpf_ctlq_desc *desc; - int err = 0; - u16 i; - - /* take the lock before we start messing with the ring */ - spin_lock(&cq->cq_lock); - - ntc = cq->next_to_clean; - - num_to_clean = *num_q_msg; - - for (i = 0; i < num_to_clean; i++) { - /* Fetch next descriptor and check if marked as done */ - desc = IDPF_CTLQ_DESC(cq, ntc); - flags = le16_to_cpu(desc->flags); - - if (!(flags & IDPF_CTLQ_FLAG_DD)) - break; - - /* Ensure no other fields are read until DD flag is checked */ - dma_rmb(); - - q_msg[i].vmvf_type = (flags & - (IDPF_CTLQ_FLAG_FTYPE_VM | - IDPF_CTLQ_FLAG_FTYPE_PF)) >> - IDPF_CTLQ_FLAG_FTYPE_S; - - if (flags & IDPF_CTLQ_FLAG_ERR) - err = -EBADMSG; - - q_msg[i].cookie.mbx.chnl_opcode = - le32_to_cpu(desc->v_opcode_dtype); - q_msg[i].cookie.mbx.chnl_retval = - le32_to_cpu(desc->v_retval); - - q_msg[i].opcode = le16_to_cpu(desc->opcode); - q_msg[i].data_len = le16_to_cpu(desc->datalen); - q_msg[i].status = le16_to_cpu(desc->ret_val); - - if (desc->datalen) { - memcpy(q_msg[i].ctx.indirect.context, - &desc->params.indirect, IDPF_INDIRECT_CTX_SIZE); - - /* Assign pointer to dma buffer to ctlq_msg array - * to be given to upper layer - */ - q_msg[i].ctx.indirect.payload = cq->bi.rx_buff[ntc]; - - /* Zero out pointer to DMA buffer info; - * will be repopulated by post buffers API - */ - cq->bi.rx_buff[ntc] = NULL; - } else { - memcpy(q_msg[i].ctx.direct, desc->params.raw, - IDPF_DIRECT_CTX_SIZE); - } - - /* Zero out stale data in descriptor */ - memset(desc, 0, sizeof(struct idpf_ctlq_desc)); - - ntc++; - if (ntc == cq->ring_size) - ntc = 0; - } - - cq->next_to_clean = ntc; - - spin_unlock(&cq->cq_lock); - - *num_q_msg = i; - if (*num_q_msg == 0) - err = -ENOMSG; - - return err; -} diff --git a/drivers/net/ethernet/intel/idpf/idpf_controlq.h b/drivers/net/ethernet/intel/idpf/idpf_controlq.h deleted file mode 100644 index de4ece40c2ff..000000000000 --- a/drivers/net/ethernet/intel/idpf/idpf_controlq.h +++ /dev/null @@ -1,144 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2023 Intel Corporation */ - -#ifndef _IDPF_CONTROLQ_H_ -#define _IDPF_CONTROLQ_H_ - -#include - -#include "idpf_controlq_api.h" - -/* Maximum buffer length for all control queue types */ -#define IDPF_CTLQ_MAX_BUF_LEN 4096 - -#define IDPF_CTLQ_DESC(R, i) \ - (&(((struct idpf_ctlq_desc *)((R)->desc_ring.va))[i])) - -#define IDPF_CTLQ_DESC_UNUSED(R) \ - ((u16)((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->ring_size) + \ - (R)->next_to_clean - (R)->next_to_use - 1)) - -/* Control Queue default settings */ -#define IDPF_CTRL_SQ_CMD_TIMEOUT 250 /* msecs */ - -struct idpf_ctlq_desc { - /* Control queue descriptor flags */ - __le16 flags; - /* Control queue message opcode */ - __le16 opcode; - __le16 datalen; /* 0 for direct commands */ - union { - __le16 ret_val; - __le16 pfid_vfid; -#define IDPF_CTLQ_DESC_VF_ID_S 0 -#define IDPF_CTLQ_DESC_VF_ID_M (0x7FF << IDPF_CTLQ_DESC_VF_ID_S) -#define IDPF_CTLQ_DESC_PF_ID_S 11 -#define IDPF_CTLQ_DESC_PF_ID_M (0x1F << IDPF_CTLQ_DESC_PF_ID_S) - }; - - /* Virtchnl message opcode and virtchnl descriptor type - * v_opcode=[27:0], v_dtype=[31:28] - */ - __le32 v_opcode_dtype; - /* Virtchnl return value */ - __le32 v_retval; - union { - struct { - __le32 param0; - __le32 param1; - __le32 param2; - __le32 param3; - } direct; - struct { - __le32 param0; - __le16 sw_cookie; - /* Virtchnl flags */ - __le16 v_flags; - __le32 addr_high; - __le32 addr_low; - } indirect; - u8 raw[16]; - } params; -}; - -/* Flags sub-structure - * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 | - * |DD |CMP|ERR| * RSV * |FTYPE | *RSV* |RD |VFC|BUF| HOST_ID | - */ -/* command flags and offsets */ -#define IDPF_CTLQ_FLAG_DD_S 0 -#define IDPF_CTLQ_FLAG_CMP_S 1 -#define IDPF_CTLQ_FLAG_ERR_S 2 -#define IDPF_CTLQ_FLAG_FTYPE_S 6 -#define IDPF_CTLQ_FLAG_RD_S 10 -#define IDPF_CTLQ_FLAG_VFC_S 11 -#define IDPF_CTLQ_FLAG_BUF_S 12 -#define IDPF_CTLQ_FLAG_HOST_ID_S 13 - -#define IDPF_CTLQ_FLAG_DD BIT(IDPF_CTLQ_FLAG_DD_S) /* 0x1 */ -#define IDPF_CTLQ_FLAG_CMP BIT(IDPF_CTLQ_FLAG_CMP_S) /* 0x2 */ -#define IDPF_CTLQ_FLAG_ERR BIT(IDPF_CTLQ_FLAG_ERR_S) /* 0x4 */ -#define IDPF_CTLQ_FLAG_FTYPE_VM BIT(IDPF_CTLQ_FLAG_FTYPE_S) /* 0x40 */ -#define IDPF_CTLQ_FLAG_FTYPE_PF BIT(IDPF_CTLQ_FLAG_FTYPE_S + 1) /* 0x80 */ -#define IDPF_CTLQ_FLAG_RD BIT(IDPF_CTLQ_FLAG_RD_S) /* 0x400 */ -#define IDPF_CTLQ_FLAG_VFC BIT(IDPF_CTLQ_FLAG_VFC_S) /* 0x800 */ -#define IDPF_CTLQ_FLAG_BUF BIT(IDPF_CTLQ_FLAG_BUF_S) /* 0x1000 */ - -/* Host ID is a special field that has 3b and not a 1b flag */ -#define IDPF_CTLQ_FLAG_HOST_ID_M MAKE_MASK(0x7000UL, IDPF_CTLQ_FLAG_HOST_ID_S) - -struct idpf_mbxq_desc { - u8 pad[8]; /* CTLQ flags/opcode/len/retval fields */ - u32 chnl_opcode; /* avoid confusion with desc->opcode */ - u32 chnl_retval; /* ditto for desc->retval */ - u32 pf_vf_id; /* used by CP when sending to PF */ -}; - -/* Max number of MMIO regions not including the mailbox and rstat regions in - * the fallback case when the whole bar is mapped. - */ -#define IDPF_MMIO_MAP_FALLBACK_MAX_REMAINING 3 - -struct idpf_mmio_reg { - void __iomem *vaddr; - resource_size_t addr_start; - resource_size_t addr_len; -}; - -/* Define the driver hardware struct to replace other control structs as needed - * Align to ctlq_hw_info - */ -struct idpf_hw { - struct idpf_mmio_reg mbx; - struct idpf_mmio_reg rstat; - /* Array of remaining LAN BAR regions */ - int num_lan_regs; - struct idpf_mmio_reg *lan_regs; - - struct idpf_adapter *back; - - /* control queue - send and receive */ - struct idpf_ctlq_info *asq; - struct idpf_ctlq_info *arq; - - /* pci info */ - u16 device_id; - u16 vendor_id; - u16 subsystem_device_id; - u16 subsystem_vendor_id; - u8 revision_id; - bool adapter_stopped; - - struct list_head cq_list_head; -}; - -int idpf_ctlq_alloc_ring_res(struct idpf_hw *hw, - struct idpf_ctlq_info *cq); - -void idpf_ctlq_dealloc_ring_res(struct idpf_hw *hw, struct idpf_ctlq_info *cq); - -/* prototype for functions used for dynamic memory allocation */ -void *idpf_alloc_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem, - u64 size); -void idpf_free_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem); -#endif /* _IDPF_CONTROLQ_H_ */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_controlq_api.h b/drivers/net/ethernet/intel/idpf/idpf_controlq_api.h deleted file mode 100644 index 3414c5f9a831..000000000000 --- a/drivers/net/ethernet/intel/idpf/idpf_controlq_api.h +++ /dev/null @@ -1,177 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2023 Intel Corporation */ - -#ifndef _IDPF_CONTROLQ_API_H_ -#define _IDPF_CONTROLQ_API_H_ - -#include "idpf_mem.h" - -struct idpf_hw; - -/* Used for queue init, response and events */ -enum idpf_ctlq_type { - IDPF_CTLQ_TYPE_MAILBOX_TX = 0, - IDPF_CTLQ_TYPE_MAILBOX_RX = 1, - IDPF_CTLQ_TYPE_CONFIG_TX = 2, - IDPF_CTLQ_TYPE_CONFIG_RX = 3, - IDPF_CTLQ_TYPE_EVENT_RX = 4, - IDPF_CTLQ_TYPE_RDMA_TX = 5, - IDPF_CTLQ_TYPE_RDMA_RX = 6, - IDPF_CTLQ_TYPE_RDMA_COMPL = 7 -}; - -/* Generic Control Queue Structures */ -struct idpf_ctlq_reg { - /* used for queue tracking */ - u32 head; - u32 tail; - /* Below applies only to default mb (if present) */ - u32 len; - u32 bah; - u32 bal; - u32 len_mask; - u32 len_ena_mask; - u32 head_mask; -}; - -/* Generic queue msg structure */ -struct idpf_ctlq_msg { - u8 vmvf_type; /* represents the source of the message on recv */ -#define IDPF_VMVF_TYPE_VF 0 -#define IDPF_VMVF_TYPE_VM 1 -#define IDPF_VMVF_TYPE_PF 2 - u8 host_id; - /* 3b field used only when sending a message to CP - to be used in - * combination with target func_id to route the message - */ -#define IDPF_HOST_ID_MASK 0x7 - - u16 opcode; - u16 data_len; /* data_len = 0 when no payload is attached */ - union { - u16 func_id; /* when sending a message */ - u16 status; /* when receiving a message */ - }; - union { - struct { - u32 chnl_opcode; - u32 chnl_retval; - } mbx; - } cookie; - union { -#define IDPF_DIRECT_CTX_SIZE 16 -#define IDPF_INDIRECT_CTX_SIZE 8 - /* 16 bytes of context can be provided or 8 bytes of context - * plus the address of a DMA buffer - */ - u8 direct[IDPF_DIRECT_CTX_SIZE]; - struct { - u8 context[IDPF_INDIRECT_CTX_SIZE]; - struct idpf_dma_mem *payload; - } indirect; - struct { - u32 rsvd; - u16 data; - u16 flags; - } sw_cookie; - } ctx; -}; - -/* Generic queue info structures */ -/* MB, CONFIG and EVENT q do not have extended info */ -struct idpf_ctlq_create_info { - enum idpf_ctlq_type type; - int id; /* absolute queue offset passed as input - * -1 for default mailbox if present - */ - u16 len; /* Queue length passed as input */ - u16 buf_size; /* buffer size passed as input */ - u64 base_address; /* output, HPA of the Queue start */ - struct idpf_ctlq_reg reg; /* registers accessed by ctlqs */ - - int ext_info_size; - void *ext_info; /* Specific to q type */ -}; - -/* Control Queue information */ -struct idpf_ctlq_info { - struct list_head cq_list; - - enum idpf_ctlq_type cq_type; - int q_id; - spinlock_t cq_lock; /* control queue lock */ - /* used for interrupt processing */ - u16 next_to_use; - u16 next_to_clean; - u16 next_to_post; /* starting descriptor to post buffers - * to after recev - */ - - struct idpf_dma_mem desc_ring; /* descriptor ring memory - * idpf_dma_mem is defined in OSdep.h - */ - union { - struct idpf_dma_mem **rx_buff; - struct idpf_ctlq_msg **tx_msg; - } bi; - - u16 buf_size; /* queue buffer size */ - u16 ring_size; /* Number of descriptors */ - struct idpf_ctlq_reg reg; /* registers accessed by ctlqs */ -}; - -/** - * enum idpf_mbx_opc - PF/VF mailbox commands - * @idpf_mbq_opc_send_msg_to_cp: used by PF or VF to send a message to its CP - * @idpf_mbq_opc_send_msg_to_peer_drv: used by PF or VF to send a message to - * any peer driver - */ -enum idpf_mbx_opc { - idpf_mbq_opc_send_msg_to_cp = 0x0801, - idpf_mbq_opc_send_msg_to_peer_drv = 0x0804, -}; - -/* API supported for control queue management */ -/* Will init all required q including default mb. "q_info" is an array of - * create_info structs equal to the number of control queues to be created. - */ -int idpf_ctlq_init(struct idpf_hw *hw, u8 num_q, - struct idpf_ctlq_create_info *q_info); - -/* Allocate and initialize a single control queue, which will be added to the - * control queue list; returns a handle to the created control queue - */ -int idpf_ctlq_add(struct idpf_hw *hw, - struct idpf_ctlq_create_info *qinfo, - struct idpf_ctlq_info **cq); - -/* Deinitialize and deallocate a single control queue */ -void idpf_ctlq_remove(struct idpf_hw *hw, - struct idpf_ctlq_info *cq); - -/* Sends messages to HW and will also free the buffer*/ -int idpf_ctlq_send(struct idpf_hw *hw, - struct idpf_ctlq_info *cq, - u16 num_q_msg, - struct idpf_ctlq_msg q_msg[]); - -/* Receives messages and called by interrupt handler/polling - * initiated by app/process. Also caller is supposed to free the buffers - */ -int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg, - struct idpf_ctlq_msg *q_msg); - -/* Reclaims send descriptors on HW write back */ -int idpf_ctlq_clean_sq(struct idpf_ctlq_info *cq, u16 *clean_count, - struct idpf_ctlq_msg *msg_status[]); - -/* Indicate RX buffers are done being processed */ -int idpf_ctlq_post_rx_buffs(struct idpf_hw *hw, - struct idpf_ctlq_info *cq, - u16 *buff_count, - struct idpf_dma_mem **buffs); - -/* Will destroy all q including the default mb */ -void idpf_ctlq_deinit(struct idpf_hw *hw); - -#endif /* _IDPF_CONTROLQ_API_H_ */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_controlq_setup.c b/drivers/net/ethernet/intel/idpf/idpf_controlq_setup.c deleted file mode 100644 index d4d488c7cfd6..000000000000 --- a/drivers/net/ethernet/intel/idpf/idpf_controlq_setup.c +++ /dev/null @@ -1,169 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright (C) 2023 Intel Corporation */ - -#include "idpf_controlq.h" - -/** - * idpf_ctlq_alloc_desc_ring - Allocate Control Queue (CQ) rings - * @hw: pointer to hw struct - * @cq: pointer to the specific Control queue - */ -static int idpf_ctlq_alloc_desc_ring(struct idpf_hw *hw, - struct idpf_ctlq_info *cq) -{ - size_t size = cq->ring_size * sizeof(struct idpf_ctlq_desc); - - cq->desc_ring.va = idpf_alloc_dma_mem(hw, &cq->desc_ring, size); - if (!cq->desc_ring.va) - return -ENOMEM; - - return 0; -} - -/** - * idpf_ctlq_alloc_bufs - Allocate Control Queue (CQ) buffers - * @hw: pointer to hw struct - * @cq: pointer to the specific Control queue - * - * Allocate the buffer head for all control queues, and if it's a receive - * queue, allocate DMA buffers - */ -static int idpf_ctlq_alloc_bufs(struct idpf_hw *hw, - struct idpf_ctlq_info *cq) -{ - int i; - - /* Do not allocate DMA buffers for transmit queues */ - if (cq->cq_type == IDPF_CTLQ_TYPE_MAILBOX_TX) - return 0; - - /* We'll be allocating the buffer info memory first, then we can - * allocate the mapped buffers for the event processing - */ - cq->bi.rx_buff = kzalloc_objs(struct idpf_dma_mem *, cq->ring_size); - if (!cq->bi.rx_buff) - return -ENOMEM; - - /* allocate the mapped buffers (except for the last one) */ - for (i = 0; i < cq->ring_size - 1; i++) { - struct idpf_dma_mem *bi; - int num = 1; /* number of idpf_dma_mem to be allocated */ - - cq->bi.rx_buff[i] = kzalloc_objs(struct idpf_dma_mem, num); - if (!cq->bi.rx_buff[i]) - goto unwind_alloc_cq_bufs; - - bi = cq->bi.rx_buff[i]; - - bi->va = idpf_alloc_dma_mem(hw, bi, cq->buf_size); - if (!bi->va) { - /* unwind will not free the failed entry */ - kfree(cq->bi.rx_buff[i]); - goto unwind_alloc_cq_bufs; - } - } - - return 0; - -unwind_alloc_cq_bufs: - /* don't try to free the one that failed... */ - i--; - for (; i >= 0; i--) { - idpf_free_dma_mem(hw, cq->bi.rx_buff[i]); - kfree(cq->bi.rx_buff[i]); - } - kfree(cq->bi.rx_buff); - - return -ENOMEM; -} - -/** - * idpf_ctlq_free_desc_ring - Free Control Queue (CQ) rings - * @hw: pointer to hw struct - * @cq: pointer to the specific Control queue - * - * This assumes the posted send buffers have already been cleaned - * and de-allocated - */ -static void idpf_ctlq_free_desc_ring(struct idpf_hw *hw, - struct idpf_ctlq_info *cq) -{ - idpf_free_dma_mem(hw, &cq->desc_ring); -} - -/** - * idpf_ctlq_free_bufs - Free CQ buffer info elements - * @hw: pointer to hw struct - * @cq: pointer to the specific Control queue - * - * Free the DMA buffers for RX queues, and DMA buffer header for both RX and TX - * queues. The upper layers are expected to manage freeing of TX DMA buffers - */ -static void idpf_ctlq_free_bufs(struct idpf_hw *hw, struct idpf_ctlq_info *cq) -{ - void *bi; - - if (cq->cq_type == IDPF_CTLQ_TYPE_MAILBOX_RX) { - int i; - - /* free DMA buffers for rx queues*/ - for (i = 0; i < cq->ring_size; i++) { - if (cq->bi.rx_buff[i]) { - idpf_free_dma_mem(hw, cq->bi.rx_buff[i]); - kfree(cq->bi.rx_buff[i]); - } - } - - bi = (void *)cq->bi.rx_buff; - } else { - bi = (void *)cq->bi.tx_msg; - } - - /* free the buffer header */ - kfree(bi); -} - -/** - * idpf_ctlq_dealloc_ring_res - Free memory allocated for control queue - * @hw: pointer to hw struct - * @cq: pointer to the specific Control queue - * - * Free the memory used by the ring, buffers and other related structures - */ -void idpf_ctlq_dealloc_ring_res(struct idpf_hw *hw, struct idpf_ctlq_info *cq) -{ - /* free ring buffers and the ring itself */ - idpf_ctlq_free_bufs(hw, cq); - idpf_ctlq_free_desc_ring(hw, cq); -} - -/** - * idpf_ctlq_alloc_ring_res - allocate memory for descriptor ring and bufs - * @hw: pointer to hw struct - * @cq: pointer to control queue struct - * - * Do *NOT* hold cq_lock when calling this as the memory allocation routines - * called are not going to be atomic context safe - */ -int idpf_ctlq_alloc_ring_res(struct idpf_hw *hw, struct idpf_ctlq_info *cq) -{ - int err; - - /* allocate the ring memory */ - err = idpf_ctlq_alloc_desc_ring(hw, cq); - if (err) - return err; - - /* allocate buffers in the rings */ - err = idpf_ctlq_alloc_bufs(hw, cq); - if (err) - goto idpf_init_cq_free_ring; - - /* success! */ - return 0; - -idpf_init_cq_free_ring: - idpf_free_dma_mem(hw, &cq->desc_ring); - - return err; -} diff --git a/drivers/net/ethernet/intel/idpf/idpf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_dev.c index 4079a787657f..083cf6319d26 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_dev.c @@ -10,45 +10,32 @@ /** * idpf_ctlq_reg_init - initialize default mailbox registers - * @adapter: adapter structure - * @cq: pointer to the array of create control queues + * @mmio: struct that contains MMIO region info + * @cci: struct where the register offset pointer to be copied to */ -static void idpf_ctlq_reg_init(struct idpf_adapter *adapter, - struct idpf_ctlq_create_info *cq) +static void idpf_ctlq_reg_init(struct libie_mmio_info *mmio, + struct libie_ctlq_create_info *cci) { - resource_size_t mbx_start = adapter->dev_ops.static_reg_info[0].start; - int i; + struct libie_ctlq_reg *tx_reg = &cci[LIBIE_CTLQ_TYPE_TX].reg; + struct libie_ctlq_reg *rx_reg = &cci[LIBIE_CTLQ_TYPE_RX].reg; - for (i = 0; i < IDPF_NUM_DFLT_MBX_Q; i++) { - struct idpf_ctlq_create_info *ccq = cq + i; + tx_reg->head = libie_pci_get_mmio_addr(mmio, PF_FW_ATQH); + tx_reg->tail = libie_pci_get_mmio_addr(mmio, PF_FW_ATQT); + tx_reg->len = libie_pci_get_mmio_addr(mmio, PF_FW_ATQLEN); + tx_reg->addr_high = libie_pci_get_mmio_addr(mmio, PF_FW_ATQBAH); + tx_reg->addr_low = libie_pci_get_mmio_addr(mmio, PF_FW_ATQBAL); + tx_reg->len_mask = PF_FW_ATQLEN_ATQLEN_M; + tx_reg->len_ena_mask = PF_FW_ATQLEN_ATQENABLE_M; + tx_reg->head_mask = PF_FW_ATQH_ATQH_M; - switch (ccq->type) { - case IDPF_CTLQ_TYPE_MAILBOX_TX: - /* set head and tail registers in our local struct */ - ccq->reg.head = PF_FW_ATQH - mbx_start; - ccq->reg.tail = PF_FW_ATQT - mbx_start; - ccq->reg.len = PF_FW_ATQLEN - mbx_start; - ccq->reg.bah = PF_FW_ATQBAH - mbx_start; - ccq->reg.bal = PF_FW_ATQBAL - mbx_start; - ccq->reg.len_mask = PF_FW_ATQLEN_ATQLEN_M; - ccq->reg.len_ena_mask = PF_FW_ATQLEN_ATQENABLE_M; - ccq->reg.head_mask = PF_FW_ATQH_ATQH_M; - break; - case IDPF_CTLQ_TYPE_MAILBOX_RX: - /* set head and tail registers in our local struct */ - ccq->reg.head = PF_FW_ARQH - mbx_start; - ccq->reg.tail = PF_FW_ARQT - mbx_start; - ccq->reg.len = PF_FW_ARQLEN - mbx_start; - ccq->reg.bah = PF_FW_ARQBAH - mbx_start; - ccq->reg.bal = PF_FW_ARQBAL - mbx_start; - ccq->reg.len_mask = PF_FW_ARQLEN_ARQLEN_M; - ccq->reg.len_ena_mask = PF_FW_ARQLEN_ARQENABLE_M; - ccq->reg.head_mask = PF_FW_ARQH_ARQH_M; - break; - default: - break; - } - } + rx_reg->head = libie_pci_get_mmio_addr(mmio, PF_FW_ARQH); + rx_reg->tail = libie_pci_get_mmio_addr(mmio, PF_FW_ARQT); + rx_reg->len = libie_pci_get_mmio_addr(mmio, PF_FW_ARQLEN); + rx_reg->addr_high = libie_pci_get_mmio_addr(mmio, PF_FW_ARQBAH); + rx_reg->addr_low = libie_pci_get_mmio_addr(mmio, PF_FW_ARQBAL); + rx_reg->len_mask = PF_FW_ARQLEN_ARQLEN_M; + rx_reg->len_ena_mask = PF_FW_ARQLEN_ARQENABLE_M; + rx_reg->head_mask = PF_FW_ARQH_ARQH_M; } /** @@ -57,13 +44,14 @@ static void idpf_ctlq_reg_init(struct idpf_adapter *adapter, */ static void idpf_mb_intr_reg_init(struct idpf_adapter *adapter) { + struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; struct idpf_intr_reg *intr = &adapter->mb_vector.intr_reg; u32 dyn_ctl = le32_to_cpu(adapter->caps.mailbox_dyn_ctl); - intr->dyn_ctl = idpf_get_reg_addr(adapter, dyn_ctl); + intr->dyn_ctl = libie_pci_get_mmio_addr(mmio, dyn_ctl); intr->dyn_ctl_intena_m = PF_GLINT_DYN_CTL_INTENA_M; intr->dyn_ctl_itridx_m = PF_GLINT_DYN_CTL_ITR_INDX_M; - intr->icr_ena = idpf_get_reg_addr(adapter, PF_INT_DIR_OICR_ENA); + intr->icr_ena = libie_pci_get_mmio_addr(mmio, PF_INT_DIR_OICR_ENA); intr->icr_ena_ctlq_m = PF_INT_DIR_OICR_ENA_M; } @@ -78,6 +66,7 @@ static int idpf_intr_reg_init(struct idpf_vport *vport, struct idpf_adapter *adapter = vport->adapter; u16 num_vecs = rsrc->num_q_vectors; struct idpf_vec_regs *reg_vals; + struct libie_mmio_info *mmio; int num_regs, i, err = 0; u32 rx_itr, tx_itr, val; u16 total_vecs; @@ -93,14 +82,17 @@ static int idpf_intr_reg_init(struct idpf_vport *vport, goto free_reg_vals; } + mmio = &adapter->ctlq_ctx.mmio_info; + for (i = 0; i < num_vecs; i++) { struct idpf_q_vector *q_vector = &rsrc->q_vectors[i]; u16 vec_id = rsrc->q_vector_idxs[i] - IDPF_MBX_Q_VEC; struct idpf_intr_reg *intr = &q_vector->intr_reg; + struct idpf_vec_regs *reg = ®_vals[vec_id]; u32 spacing; - intr->dyn_ctl = idpf_get_reg_addr(adapter, - reg_vals[vec_id].dyn_ctl_reg); + intr->dyn_ctl = libie_pci_get_mmio_addr(mmio, + reg->dyn_ctl_reg); intr->dyn_ctl_intena_m = PF_GLINT_DYN_CTL_INTENA_M; intr->dyn_ctl_intena_msk_m = PF_GLINT_DYN_CTL_INTENA_MSK_M; intr->dyn_ctl_itridx_s = PF_GLINT_DYN_CTL_ITR_INDX_S; @@ -110,22 +102,21 @@ static int idpf_intr_reg_init(struct idpf_vport *vport, intr->dyn_ctl_sw_itridx_ena_m = PF_GLINT_DYN_CTL_SW_ITR_INDX_ENA_M; - spacing = IDPF_ITR_IDX_SPACING(reg_vals[vec_id].itrn_index_spacing, + spacing = IDPF_ITR_IDX_SPACING(reg->itrn_index_spacing, IDPF_PF_ITR_IDX_SPACING); rx_itr = PF_GLINT_ITR_ADDR(VIRTCHNL2_ITR_IDX_0, - reg_vals[vec_id].itrn_reg, - spacing); + reg->itrn_reg, spacing); tx_itr = PF_GLINT_ITR_ADDR(VIRTCHNL2_ITR_IDX_1, - reg_vals[vec_id].itrn_reg, - spacing); - intr->rx_itr = idpf_get_reg_addr(adapter, rx_itr); - intr->tx_itr = idpf_get_reg_addr(adapter, tx_itr); + reg->itrn_reg, spacing); + intr->rx_itr = libie_pci_get_mmio_addr(mmio, rx_itr); + intr->tx_itr = libie_pci_get_mmio_addr(mmio, tx_itr); } /* Data vector for NOIRQ queues */ val = reg_vals[rsrc->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; - rsrc->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); + rsrc->noirq_dyn_ctl = + libie_pci_get_mmio_addr(&adapter->ctlq_ctx.mmio_info, val); val = PF_GLINT_DYN_CTL_WB_ON_ITR_M | PF_GLINT_DYN_CTL_INTENA_MSK_M | FIELD_PREP(PF_GLINT_DYN_CTL_ITR_INDX_M, IDPF_NO_ITR_UPDATE_IDX); @@ -143,7 +134,9 @@ static int idpf_intr_reg_init(struct idpf_vport *vport, */ static void idpf_reset_reg_init(struct idpf_adapter *adapter) { - adapter->reset_reg.rstat = idpf_get_rstat_reg_addr(adapter, PFGEN_RSTAT); + adapter->reset_reg.rstat = + libie_pci_get_mmio_addr(&adapter->ctlq_ctx.mmio_info, + PFGEN_RSTAT); adapter->reset_reg.rstat_m = PFGEN_RSTAT_PFR_STATE_M; } @@ -155,11 +148,11 @@ static void idpf_reset_reg_init(struct idpf_adapter *adapter) static void idpf_trigger_reset(struct idpf_adapter *adapter, enum idpf_flags __always_unused trig_cause) { - u32 reset_reg; + void __iomem *addr; - reset_reg = readl(idpf_get_rstat_reg_addr(adapter, PFGEN_CTRL)); - writel(reset_reg | PFGEN_CTRL_PFSWR, - idpf_get_rstat_reg_addr(adapter, PFGEN_CTRL)); + addr = libie_pci_get_mmio_addr(&adapter->ctlq_ctx.mmio_info, + PFGEN_CTRL); + writel(readl(addr) | PFGEN_CTRL_PFSWR, addr); } /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index bb99d9e7c65d..95c45f12b0f9 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -225,7 +225,7 @@ static int idpf_add_flow_steer(struct net_device *netdev, spin_unlock_bh(&vport_config->flow_steer_list_lock); if (err) - goto out; + goto out_free_fltr; rule->vport_id = cpu_to_le32(vport->vport_id); rule->count = cpu_to_le32(1); @@ -252,17 +252,15 @@ static int idpf_add_flow_steer(struct net_device *netdev, break; default: err = -EINVAL; - goto out; + goto out_free_fltr; } err = idpf_add_del_fsteer_filters(vport->adapter, rule, VIRTCHNL2_OP_ADD_FLOW_RULE); - if (err) - goto out; - - if (info->status != cpu_to_le32(VIRTCHNL2_FLOW_RULE_SUCCESS)) { - err = -EIO; - goto out; + if (err) { + /* virtchnl2 rule is already consumed */ + kfree(fltr); + return err; } /* Save a copy of the user's flow spec so ethtool can later retrieve it */ @@ -274,9 +272,10 @@ static int idpf_add_flow_steer(struct net_device *netdev, user_config->num_fsteer_fltrs++; spin_unlock_bh(&vport_config->flow_steer_list_lock); - goto out_free_rule; -out: + return 0; + +out_free_fltr: kfree(fltr); out_free_rule: kfree(rule); @@ -319,12 +318,7 @@ static int idpf_del_flow_steer(struct net_device *netdev, err = idpf_add_del_fsteer_filters(vport->adapter, rule, VIRTCHNL2_OP_DEL_FLOW_RULE); if (err) - goto out; - - if (info->status != cpu_to_le32(VIRTCHNL2_FLOW_RULE_SUCCESS)) { - err = -EIO; - goto out; - } + return err; spin_lock_bh(&vport_config->flow_steer_list_lock); list_for_each_entry_safe(f, iter, @@ -340,8 +334,6 @@ static int idpf_del_flow_steer(struct net_device *netdev, out_unlock: spin_unlock_bh(&vport_config->flow_steer_list_lock); -out: - kfree(rule); return err; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_idc.c b/drivers/net/ethernet/intel/idpf/idpf_idc.c index b7d6b08fc89e..b6cd1c25ae5d 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_idc.c +++ b/drivers/net/ethernet/intel/idpf/idpf_idc.c @@ -416,9 +416,12 @@ idpf_idc_init_msix_data(struct idpf_adapter *adapter) int idpf_idc_init_aux_core_dev(struct idpf_adapter *adapter, enum iidc_function_type ftype) { + struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; struct iidc_rdma_core_dev_info *cdev_info; struct iidc_rdma_priv_dev_info *privd; - int err, i; + struct libie_pci_mmio_region *mr; + size_t num_mem_regions; + int err, i = 0; adapter->cdev_info = kzalloc_obj(*cdev_info); if (!adapter->cdev_info) @@ -436,22 +439,31 @@ int idpf_idc_init_aux_core_dev(struct idpf_adapter *adapter, cdev_info->rdma_protocol = IIDC_RDMA_PROTOCOL_ROCEV2; privd->ftype = ftype; + num_mem_regions = list_count_nodes(&mmio->mmio_list); + if (num_mem_regions <= IDPF_MMIO_REG_NUM_STATIC) { + err = -EINVAL; + goto err_plug_aux_dev; + } + + num_mem_regions -= IDPF_MMIO_REG_NUM_STATIC; privd->mapped_mem_regions = kzalloc_objs(struct iidc_rdma_lan_mapped_mem_region, - adapter->hw.num_lan_regs); + num_mem_regions); if (!privd->mapped_mem_regions) { err = -ENOMEM; goto err_plug_aux_dev; } - privd->num_memory_regions = cpu_to_le16(adapter->hw.num_lan_regs); - for (i = 0; i < adapter->hw.num_lan_regs; i++) { - privd->mapped_mem_regions[i].region_addr = - adapter->hw.lan_regs[i].vaddr; - privd->mapped_mem_regions[i].size = - cpu_to_le64(adapter->hw.lan_regs[i].addr_len); - privd->mapped_mem_regions[i].start_offset = - cpu_to_le64(adapter->hw.lan_regs[i].addr_start); + privd->num_memory_regions = cpu_to_le16(num_mem_regions); + list_for_each_entry(mr, &mmio->mmio_list, list) { + if (!idpf_mmio_region_non_static(&adapter->ctlq_ctx.mmio_info, + mr)) + continue; + + privd->mapped_mem_regions[i].region_addr = mr->addr; + privd->mapped_mem_regions[i].size = cpu_to_le64(mr->size); + privd->mapped_mem_regions[i++].start_offset = + cpu_to_le64(mr->offset); } idpf_idc_init_msix_data(adapter); diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index bb81e620c5c8..827c795afcb6 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -68,9 +68,11 @@ static void idpf_deinit_vector_stack(struct idpf_adapter *adapter) * This will also disable interrupt mode and queue up mailbox task. Mailbox * task will reschedule itself if not in interrupt mode. */ -static void idpf_mb_intr_rel_irq(struct idpf_adapter *adapter) +void idpf_mb_intr_rel_irq(struct idpf_adapter *adapter) { - clear_bit(IDPF_MB_INTR_MODE, adapter->flags); + if (!test_and_clear_bit(IDPF_MB_INTR_MODE, adapter->flags)) + return; + kfree(free_irq(adapter->msix_entries[0].vector, adapter)); queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0); } @@ -1109,8 +1111,6 @@ static void idpf_vport_rel(struct idpf_vport *vport) kfree(adapter->vport_params_recvd[idx]); adapter->vport_params_recvd[idx] = NULL; - kfree(adapter->vport_params_reqd[idx]); - adapter->vport_params_reqd[idx] = NULL; kfree(vport); adapter->num_alloc_vports--; @@ -1373,6 +1373,7 @@ void idpf_statistics_task(struct work_struct *work) */ void idpf_mbx_task(struct work_struct *work) { + struct libie_ctlq_xn_recv_params xn_params; struct idpf_adapter *adapter; adapter = container_of(work, struct idpf_adapter, mbx_task.work); @@ -1383,7 +1384,14 @@ void idpf_mbx_task(struct work_struct *work) queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, usecs_to_jiffies(300)); - idpf_recv_mb_msg(adapter, adapter->hw.arq); + xn_params = (struct libie_ctlq_xn_recv_params) { + .xnm = adapter->xnm, + .ctlq = adapter->arq, + .ctlq_msg_handler = idpf_recv_event_msg, + .budget = LIBIE_CTLQ_MAX_XN_ENTRIES, + }; + + libie_ctlq_xn_recv(&xn_params); } /** @@ -1849,15 +1857,14 @@ void idpf_deinit_task(struct idpf_adapter *adapter) /** * idpf_check_reset_complete - check that reset is complete - * @hw: pointer to hw struct + * @adapter: adapter to check * @reset_reg: struct with reset registers * * Returns 0 if device is ready to use, or -EBUSY if it's in reset. **/ -static int idpf_check_reset_complete(struct idpf_hw *hw, +static int idpf_check_reset_complete(struct idpf_adapter *adapter, struct idpf_reset_reg *reset_reg) { - struct idpf_adapter *adapter = hw->back; int i; for (i = 0; i < 2000; i++) { @@ -1920,7 +1927,7 @@ static void idpf_init_hard_reset(struct idpf_adapter *adapter) } /* Wait for reset to complete */ - err = idpf_check_reset_complete(&adapter->hw, &adapter->reset_reg); + err = idpf_check_reset_complete(adapter, &adapter->reset_reg); if (err) { dev_err(dev, "The driver was unable to contact the device's firmware. Check that the FW is running. Driver state= 0x%x\n", adapter->state); @@ -1934,14 +1941,11 @@ static void idpf_init_hard_reset(struct idpf_adapter *adapter) goto unlock_mutex; } - queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0); - /* Initialize the state machine, also allocate memory and request * resources */ err = idpf_vc_core_init(adapter); if (err) { - cancel_delayed_work_sync(&adapter->mbx_task); idpf_deinit_dflt_mbx(adapter); goto unlock_mutex; } @@ -1987,7 +1991,8 @@ void idpf_vc_event_task(struct work_struct *work) return; func_reset: - idpf_vc_xn_shutdown(adapter->vcxn_mngr); + if (adapter->xnm) + libie_ctlq_xn_shutdown(adapter->xnm); drv_load: set_bit(IDPF_HR_RESET_IN_PROG, adapter->flags); idpf_init_hard_reset(adapter); @@ -2570,44 +2575,6 @@ static int idpf_set_mac(struct net_device *netdev, void *p) return err; } -/** - * idpf_alloc_dma_mem - Allocate dma memory - * @hw: pointer to hw struct - * @mem: pointer to dma_mem struct - * @size: size of the memory to allocate - */ -void *idpf_alloc_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem, u64 size) -{ - struct idpf_adapter *adapter = hw->back; - size_t sz = ALIGN(size, 4096); - - /* The control queue resources are freed under a spinlock, contiguous - * pages will avoid IOMMU remapping and the use vmap (and vunmap in - * dma_free_*() path. - */ - mem->va = dma_alloc_attrs(&adapter->pdev->dev, sz, &mem->pa, - GFP_KERNEL, DMA_ATTR_FORCE_CONTIGUOUS); - mem->size = sz; - - return mem->va; -} - -/** - * idpf_free_dma_mem - Free the allocated dma memory - * @hw: pointer to hw struct - * @mem: pointer to dma_mem struct - */ -void idpf_free_dma_mem(struct idpf_hw *hw, struct idpf_dma_mem *mem) -{ - struct idpf_adapter *adapter = hw->back; - - dma_free_attrs(&adapter->pdev->dev, mem->size, - mem->va, mem->pa, DMA_ATTR_FORCE_CONTIGUOUS); - mem->size = 0; - mem->va = NULL; - mem->pa = 0; -} - static int idpf_hwtstamp_set(struct net_device *netdev, struct kernel_hwtstamp_config *config, struct netlink_ext_ack *extack) diff --git a/drivers/net/ethernet/intel/idpf/idpf_main.c b/drivers/net/ethernet/intel/idpf/idpf_main.c index ab3c409e587b..9840580fbe51 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_main.c +++ b/drivers/net/ethernet/intel/idpf/idpf_main.c @@ -15,6 +15,8 @@ MODULE_DESCRIPTION(DRV_SUMMARY); MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBIE_CP"); +MODULE_IMPORT_NS("LIBIE_PCI"); MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL"); @@ -56,8 +58,16 @@ static int idpf_get_device_type(struct pci_dev *pdev) static int idpf_dev_init(struct idpf_adapter *adapter, const struct pci_device_id *ent) { + struct libie_mmio_info *mmio_info = &adapter->ctlq_ctx.mmio_info; int ret; + ret = libie_pci_init_dev(adapter->pdev); + if (ret) + return ret; + + mmio_info->pdev = adapter->pdev; + INIT_LIST_HEAD(&mmio_info->mmio_list); + if (ent->class == IDPF_CLASS_NETWORK_ETHERNET_PROGIF) { ret = idpf_get_device_type(adapter->pdev); switch (ret) { @@ -90,6 +100,15 @@ static int idpf_dev_init(struct idpf_adapter *adapter, return 0; } +/** + * idpf_decfg_device - deconfigure device and device specific resources + * @adapter: driver specific private structure + */ +static void idpf_decfg_device(struct idpf_adapter *adapter) +{ + libie_pci_unmap_all_mmio_regions(&adapter->ctlq_ctx.mmio_info); +} + /** * idpf_remove - Device removal routine * @pdev: PCI device information struct @@ -151,14 +170,13 @@ static void idpf_remove(struct pci_dev *pdev) adapter->vport_config = NULL; kfree(adapter->netdevs); adapter->netdevs = NULL; - kfree(adapter->vcxn_mngr); - adapter->vcxn_mngr = NULL; mutex_destroy(&adapter->vport_ctrl_lock); mutex_destroy(&adapter->vector_lock); mutex_destroy(&adapter->queue_lock); mutex_destroy(&adapter->vc_buf_lock); + idpf_decfg_device(adapter); pci_set_drvdata(pdev, NULL); kfree(adapter); } @@ -181,46 +199,44 @@ static void idpf_shutdown(struct pci_dev *pdev) } /** - * idpf_cfg_hw - Initialize HW struct - * @adapter: adapter to setup hw struct for + * idpf_cfg_device - configure device and device specific resources + * @adapter: driver specific private structure * - * Returns 0 on success, negative on failure + * Return: %0 on success, -%errno on failure. */ -static int idpf_cfg_hw(struct idpf_adapter *adapter) +static int idpf_cfg_device(struct idpf_adapter *adapter) { - resource_size_t res_start, mbx_start, rstat_start; + struct libie_mmio_info *mmio_info = &adapter->ctlq_ctx.mmio_info; struct pci_dev *pdev = adapter->pdev; - struct idpf_hw *hw = &adapter->hw; - struct device *dev = &pdev->dev; - long len; - - res_start = pci_resource_start(pdev, 0); + struct resource *region; + bool mapped; + int err; /* Map mailbox space for virtchnl communication */ - mbx_start = res_start + adapter->dev_ops.static_reg_info[0].start; - len = resource_size(&adapter->dev_ops.static_reg_info[0]); - hw->mbx.vaddr = devm_ioremap(dev, mbx_start, len); - if (!hw->mbx.vaddr) { - pci_err(pdev, "failed to allocate BAR0 mbx region\n"); - + region = &adapter->dev_ops.static_reg_info[0]; + mapped = libie_pci_map_mmio_region(mmio_info, region->start, + resource_size(region)); + if (!mapped) { + pci_err(pdev, "failed to map BAR0 mbx region\n"); return -ENOMEM; } - hw->mbx.addr_start = adapter->dev_ops.static_reg_info[0].start; - hw->mbx.addr_len = len; /* Map rstat space for resets */ - rstat_start = res_start + adapter->dev_ops.static_reg_info[1].start; - len = resource_size(&adapter->dev_ops.static_reg_info[1]); - hw->rstat.vaddr = devm_ioremap(dev, rstat_start, len); - if (!hw->rstat.vaddr) { - pci_err(pdev, "failed to allocate BAR0 rstat region\n"); + region = &adapter->dev_ops.static_reg_info[1]; + mapped = libie_pci_map_mmio_region(mmio_info, region->start, + resource_size(region)); + if (!mapped) { + pci_err(pdev, "failed to map BAR0 rstat region\n"); + libie_pci_unmap_all_mmio_regions(mmio_info); return -ENOMEM; } - hw->rstat.addr_start = adapter->dev_ops.static_reg_info[1].start; - hw->rstat.addr_len = len; - hw->back = adapter; + err = pci_enable_ptm(pdev); + if (err) + pci_dbg(pdev, "PCIe PTM is not supported by PCIe bus/controller\n"); + + pci_set_drvdata(pdev, adapter); return 0; } @@ -246,32 +262,21 @@ static int idpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->req_rx_splitq = true; adapter->pdev = pdev; - err = pcim_enable_device(pdev); - if (err) - goto err_free; - err = pcim_request_region(pdev, 0, pci_name(pdev)); + err = idpf_dev_init(adapter, ent); if (err) { - pci_err(pdev, "pcim_request_region failed %pe\n", ERR_PTR(err)); - + dev_err(&pdev->dev, "Failed to initialize device (ID 0x%x): %d\n", + ent->device, err); goto err_free; } - err = pci_enable_ptm(pdev); - if (err) - pci_dbg(pdev, "PCIe PTM is not supported by PCIe bus/controller\n"); - - /* set up for high or low dma */ - err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + err = idpf_cfg_device(adapter); if (err) { - pci_err(pdev, "DMA configuration failed: %pe\n", ERR_PTR(err)); - + pci_err(pdev, "Failed to configure device specific resources: %pe\n", + ERR_PTR(err)); goto err_free; } - pci_set_master(pdev); - pci_set_drvdata(pdev, adapter); - adapter->init_wq = alloc_workqueue("%s-%s-init", WQ_UNBOUND | WQ_MEM_RECLAIM, 0, dev_driver_string(dev), @@ -279,7 +284,7 @@ static int idpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (!adapter->init_wq) { dev_err(dev, "Failed to allocate init workqueue\n"); err = -ENOMEM; - goto err_free; + goto err_init_wq; } adapter->serv_wq = alloc_workqueue("%s-%s-service", @@ -324,20 +329,6 @@ static int idpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* setup msglvl */ adapter->msg_enable = netif_msg_init(-1, IDPF_AVAIL_NETIF_M); - err = idpf_dev_init(adapter, ent); - if (err) { - dev_err(&pdev->dev, "Unexpected dev ID 0x%x in idpf probe\n", - ent->device); - goto destroy_vc_event_wq; - } - - err = idpf_cfg_hw(adapter); - if (err) { - dev_err(dev, "Failed to configure HW structure for adapter: %d\n", - err); - goto destroy_vc_event_wq; - } - mutex_init(&adapter->vport_ctrl_lock); mutex_init(&adapter->vector_lock); mutex_init(&adapter->queue_lock); @@ -356,8 +347,6 @@ static int idpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; -destroy_vc_event_wq: - destroy_workqueue(adapter->vc_event_wq); err_vc_event_wq_alloc: destroy_workqueue(adapter->stats_wq); err_stats_wq_alloc: @@ -366,6 +355,8 @@ static int idpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) destroy_workqueue(adapter->serv_wq); err_serv_wq_alloc: destroy_workqueue(adapter->init_wq); +err_init_wq: + idpf_decfg_device(adapter); err_free: kfree(adapter); return err; diff --git a/drivers/net/ethernet/intel/idpf/idpf_mem.h b/drivers/net/ethernet/intel/idpf/idpf_mem.h deleted file mode 100644 index 2aaabdc02dd2..000000000000 --- a/drivers/net/ethernet/intel/idpf/idpf_mem.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2023 Intel Corporation */ - -#ifndef _IDPF_MEM_H_ -#define _IDPF_MEM_H_ - -#include - -struct idpf_dma_mem { - void *va; - dma_addr_t pa; - size_t size; -}; - -#define idpf_mbx_wr32(a, reg, value) writel((value), ((a)->mbx.vaddr + (reg))) -#define idpf_mbx_rd32(a, reg) readl((a)->mbx.vaddr + (reg)) -#define idpf_mbx_wr64(a, reg, value) writeq((value), ((a)->mbx.vaddr + (reg))) -#define idpf_mbx_rd64(a, reg) readq((a)->mbx.vaddr + (reg)) - -#endif /* _IDPF_MEM_H_ */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 99fcd8e298d6..c3ffb445297f 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -4676,11 +4676,11 @@ int idpf_config_rss(struct idpf_vport *vport, struct idpf_rss_data *rss_data) u32 vport_id = vport->vport_id; int err; - err = idpf_send_get_set_rss_key_msg(adapter, rss_data, vport_id, false); + err = idpf_send_set_rss_key_msg(adapter, rss_data, vport_id); if (err) return err; - return idpf_send_get_set_rss_lut_msg(adapter, rss_data, vport_id, false); + return idpf_send_set_rss_lut_msg(adapter, rss_data, vport_id); } /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 908dfa28674e..93547597efd2 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -5,6 +5,7 @@ #define _IDPF_TXRX_H_ #include +#include #include #include @@ -13,7 +14,6 @@ #include #include "idpf_lan_txrx.h" -#include "virtchnl2_lan_desc.h" #define IDPF_LARGE_MAX_Q 256 #define IDPF_MAX_Q 16 @@ -236,7 +236,7 @@ enum idpf_tx_ctx_desc_eipt_offload { (sizeof(u16) * IDPF_RX_MAX_PTYPE_PROTO_IDS)) #define IDPF_RX_PTYPE_HDR_SZ sizeof(struct virtchnl2_get_ptype_info) #define IDPF_RX_MAX_PTYPES_PER_BUF \ - DIV_ROUND_DOWN_ULL((IDPF_CTLQ_MAX_BUF_LEN - IDPF_RX_PTYPE_HDR_SZ), \ + DIV_ROUND_DOWN_ULL(LIBIE_CTLQ_MAX_BUF_LEN - IDPF_RX_PTYPE_HDR_SZ, \ IDPF_RX_MAX_PTYPE_SZ) #define IDPF_GET_PTYPE_SIZE(p) struct_size((p), proto_id, (p)->proto_id_count) diff --git a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c index 6726084f6cfa..b537de3592f4 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c @@ -9,45 +9,32 @@ /** * idpf_vf_ctlq_reg_init - initialize default mailbox registers - * @adapter: adapter structure - * @cq: pointer to the array of create control queues + * @mmio: struct that contains MMIO region info + * @cci: struct where the register offset pointer to be copied to */ -static void idpf_vf_ctlq_reg_init(struct idpf_adapter *adapter, - struct idpf_ctlq_create_info *cq) +static void idpf_vf_ctlq_reg_init(struct libie_mmio_info *mmio, + struct libie_ctlq_create_info *cci) { - resource_size_t mbx_start = adapter->dev_ops.static_reg_info[0].start; - int i; + struct libie_ctlq_reg *tx_reg = &cci[LIBIE_CTLQ_TYPE_TX].reg; + struct libie_ctlq_reg *rx_reg = &cci[LIBIE_CTLQ_TYPE_RX].reg; - for (i = 0; i < IDPF_NUM_DFLT_MBX_Q; i++) { - struct idpf_ctlq_create_info *ccq = cq + i; + tx_reg->head = libie_pci_get_mmio_addr(mmio, VF_ATQH); + tx_reg->tail = libie_pci_get_mmio_addr(mmio, VF_ATQT); + tx_reg->len = libie_pci_get_mmio_addr(mmio, VF_ATQLEN); + tx_reg->addr_high = libie_pci_get_mmio_addr(mmio, VF_ATQBAH); + tx_reg->addr_low = libie_pci_get_mmio_addr(mmio, VF_ATQBAL); + tx_reg->len_mask = VF_ATQLEN_ATQLEN_M; + tx_reg->len_ena_mask = VF_ATQLEN_ATQENABLE_M; + tx_reg->head_mask = VF_ATQH_ATQH_M; - switch (ccq->type) { - case IDPF_CTLQ_TYPE_MAILBOX_TX: - /* set head and tail registers in our local struct */ - ccq->reg.head = VF_ATQH - mbx_start; - ccq->reg.tail = VF_ATQT - mbx_start; - ccq->reg.len = VF_ATQLEN - mbx_start; - ccq->reg.bah = VF_ATQBAH - mbx_start; - ccq->reg.bal = VF_ATQBAL - mbx_start; - ccq->reg.len_mask = VF_ATQLEN_ATQLEN_M; - ccq->reg.len_ena_mask = VF_ATQLEN_ATQENABLE_M; - ccq->reg.head_mask = VF_ATQH_ATQH_M; - break; - case IDPF_CTLQ_TYPE_MAILBOX_RX: - /* set head and tail registers in our local struct */ - ccq->reg.head = VF_ARQH - mbx_start; - ccq->reg.tail = VF_ARQT - mbx_start; - ccq->reg.len = VF_ARQLEN - mbx_start; - ccq->reg.bah = VF_ARQBAH - mbx_start; - ccq->reg.bal = VF_ARQBAL - mbx_start; - ccq->reg.len_mask = VF_ARQLEN_ARQLEN_M; - ccq->reg.len_ena_mask = VF_ARQLEN_ARQENABLE_M; - ccq->reg.head_mask = VF_ARQH_ARQH_M; - break; - default: - break; - } - } + rx_reg->head = libie_pci_get_mmio_addr(mmio, VF_ARQH); + rx_reg->tail = libie_pci_get_mmio_addr(mmio, VF_ARQT); + rx_reg->len = libie_pci_get_mmio_addr(mmio, VF_ARQLEN); + rx_reg->addr_high = libie_pci_get_mmio_addr(mmio, VF_ARQBAH); + rx_reg->addr_low = libie_pci_get_mmio_addr(mmio, VF_ARQBAL); + rx_reg->len_mask = VF_ARQLEN_ARQLEN_M; + rx_reg->len_ena_mask = VF_ARQLEN_ARQENABLE_M; + rx_reg->head_mask = VF_ARQH_ARQH_M; } /** @@ -56,13 +43,14 @@ static void idpf_vf_ctlq_reg_init(struct idpf_adapter *adapter, */ static void idpf_vf_mb_intr_reg_init(struct idpf_adapter *adapter) { + struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; struct idpf_intr_reg *intr = &adapter->mb_vector.intr_reg; u32 dyn_ctl = le32_to_cpu(adapter->caps.mailbox_dyn_ctl); - intr->dyn_ctl = idpf_get_reg_addr(adapter, dyn_ctl); + intr->dyn_ctl = libie_pci_get_mmio_addr(mmio, dyn_ctl); intr->dyn_ctl_intena_m = VF_INT_DYN_CTL0_INTENA_M; intr->dyn_ctl_itridx_m = VF_INT_DYN_CTL0_ITR_INDX_M; - intr->icr_ena = idpf_get_reg_addr(adapter, VF_INT_ICR0_ENA1); + intr->icr_ena = libie_pci_get_mmio_addr(mmio, VF_INT_ICR0_ENA1); intr->icr_ena_ctlq_m = VF_INT_ICR0_ENA1_ADMINQ_M; } @@ -77,6 +65,7 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport, struct idpf_adapter *adapter = vport->adapter; u16 num_vecs = rsrc->num_q_vectors; struct idpf_vec_regs *reg_vals; + struct libie_mmio_info *mmio; int num_regs, i, err = 0; u32 rx_itr, tx_itr, val; u16 total_vecs; @@ -92,14 +81,17 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport, goto free_reg_vals; } + mmio = &adapter->ctlq_ctx.mmio_info; + for (i = 0; i < num_vecs; i++) { struct idpf_q_vector *q_vector = &rsrc->q_vectors[i]; u16 vec_id = rsrc->q_vector_idxs[i] - IDPF_MBX_Q_VEC; struct idpf_intr_reg *intr = &q_vector->intr_reg; + struct idpf_vec_regs *reg = ®_vals[vec_id]; u32 spacing; - intr->dyn_ctl = idpf_get_reg_addr(adapter, - reg_vals[vec_id].dyn_ctl_reg); + intr->dyn_ctl = libie_pci_get_mmio_addr(mmio, + reg->dyn_ctl_reg); intr->dyn_ctl_intena_m = VF_INT_DYN_CTLN_INTENA_M; intr->dyn_ctl_intena_msk_m = VF_INT_DYN_CTLN_INTENA_MSK_M; intr->dyn_ctl_itridx_s = VF_INT_DYN_CTLN_ITR_INDX_S; @@ -109,22 +101,21 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport, intr->dyn_ctl_sw_itridx_ena_m = VF_INT_DYN_CTLN_SW_ITR_INDX_ENA_M; - spacing = IDPF_ITR_IDX_SPACING(reg_vals[vec_id].itrn_index_spacing, + spacing = IDPF_ITR_IDX_SPACING(reg->itrn_index_spacing, IDPF_VF_ITR_IDX_SPACING); rx_itr = VF_INT_ITRN_ADDR(VIRTCHNL2_ITR_IDX_0, - reg_vals[vec_id].itrn_reg, - spacing); + reg->itrn_reg, spacing); tx_itr = VF_INT_ITRN_ADDR(VIRTCHNL2_ITR_IDX_1, - reg_vals[vec_id].itrn_reg, - spacing); - intr->rx_itr = idpf_get_reg_addr(adapter, rx_itr); - intr->tx_itr = idpf_get_reg_addr(adapter, tx_itr); + reg->itrn_reg, spacing); + intr->rx_itr = libie_pci_get_mmio_addr(mmio, rx_itr); + intr->tx_itr = libie_pci_get_mmio_addr(mmio, tx_itr); } /* Data vector for NOIRQ queues */ val = reg_vals[rsrc->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; - rsrc->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); + rsrc->noirq_dyn_ctl = + libie_pci_get_mmio_addr(&adapter->ctlq_ctx.mmio_info, val); val = VF_INT_DYN_CTLN_WB_ON_ITR_M | VF_INT_DYN_CTLN_INTENA_MSK_M | FIELD_PREP(VF_INT_DYN_CTLN_ITR_INDX_M, IDPF_NO_ITR_UPDATE_IDX); @@ -142,7 +133,9 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport, */ static void idpf_vf_reset_reg_init(struct idpf_adapter *adapter) { - adapter->reset_reg.rstat = idpf_get_rstat_reg_addr(adapter, VFGEN_RSTAT); + adapter->reset_reg.rstat = + libie_pci_get_mmio_addr(&adapter->ctlq_ctx.mmio_info, + VFGEN_RSTAT); adapter->reset_reg.rstat_m = VFGEN_RSTAT_VFR_STATE_M; } @@ -157,8 +150,7 @@ static void idpf_vf_trigger_reset(struct idpf_adapter *adapter, /* Do not send VIRTCHNL2_OP_RESET_VF message on driver unload */ if (trig_cause == IDPF_HR_FUNC_RESET && !test_bit(IDPF_REMOVE_IN_PROG, adapter->flags)) - idpf_send_mb_msg(adapter, adapter->hw.asq, - VIRTCHNL2_OP_RESET_VF, 0, NULL, 0); + idpf_send_vf_reset_msg(adapter); } /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index 8bd6cca64c9b..1caf52706973 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -2,26 +2,13 @@ /* Copyright (C) 2023 Intel Corporation */ #include +#include #include #include "idpf.h" #include "idpf_virtchnl.h" #include "idpf_ptp.h" -/** - * struct idpf_vc_xn_manager - Manager for tracking transactions - * @ring: backing and lookup for transactions - * @free_xn_bm: bitmap for free transactions - * @xn_bm_lock: make bitmap access synchronous where necessary - * @salt: used to make cookie unique every message - */ -struct idpf_vc_xn_manager { - struct idpf_vc_xn ring[IDPF_VC_XN_RING_LEN]; - DECLARE_BITMAP(free_xn_bm, IDPF_VC_XN_RING_LEN); - spinlock_t xn_bm_lock; - u8 salt; -}; - /** * idpf_vid_to_vport - Translate vport id to vport pointer * @adapter: private data struct @@ -82,79 +69,62 @@ static void idpf_handle_event_link(struct idpf_adapter *adapter, /** * idpf_recv_event_msg - Receive virtchnl event message - * @adapter: Driver specific private structure + * @ctx: control queue context * @ctlq_msg: message to copy from * * Receive virtchnl event message */ -static void idpf_recv_event_msg(struct idpf_adapter *adapter, - struct idpf_ctlq_msg *ctlq_msg) +void idpf_recv_event_msg(struct libie_ctlq_ctx *ctx, + struct libie_ctlq_msg *ctlq_msg) { - int payload_size = ctlq_msg->ctx.indirect.payload->size; + struct kvec *buff = &ctlq_msg->recv_mem; + int payload_size = buff->iov_len; + struct idpf_adapter *adapter; struct virtchnl2_event *v2e; u32 event; - if (payload_size < sizeof(*v2e)) { - dev_err_ratelimited(&adapter->pdev->dev, "Failed to receive valid payload for event msg (op %d len %d)\n", - ctlq_msg->cookie.mbx.chnl_opcode, - payload_size); - return; + adapter = container_of(ctx, struct idpf_adapter, ctlq_ctx); + if (ctlq_msg->chnl_opcode != VIRTCHNL2_OP_EVENT) { + dev_dbg(&adapter->pdev->dev, + "Unhandled message with opcode %u from CP\n", + ctlq_msg->chnl_opcode); + goto free_rx_buf; } - v2e = (struct virtchnl2_event *)ctlq_msg->ctx.indirect.payload->va; + if (payload_size < sizeof(*v2e)) { + dev_err_ratelimited(&adapter->pdev->dev, "Failed to receive valid payload for event msg (op %d len %d)\n", + ctlq_msg->chnl_opcode, + payload_size); + goto free_rx_buf; + } + + v2e = (struct virtchnl2_event *)buff->iov_base; event = le32_to_cpu(v2e->event); switch (event) { case VIRTCHNL2_EVENT_LINK_CHANGE: idpf_handle_event_link(adapter, v2e); - return; + break; default: dev_err(&adapter->pdev->dev, "Unknown event %d from PF\n", event); break; } + +free_rx_buf: + libie_ctlq_release_rx_buf(buff); } /** * idpf_mb_clean - Reclaim the send mailbox queue entries - * @adapter: driver specific private structure * @asq: send control queue info + * @deinit: release all buffers before destroying the queue * - * Reclaim the send mailbox queue entries to be used to send further messages - * - * Return: 0 on success, negative on failure + * This is a helper function to clean the send mailbox queue entries. */ -static int idpf_mb_clean(struct idpf_adapter *adapter, - struct idpf_ctlq_info *asq) +static void idpf_mb_clean(struct libie_ctlq_info *asq, bool deinit) { - u16 i, num_q_msg = IDPF_DFLT_MBX_Q_LEN; - struct idpf_ctlq_msg **q_msg; - struct idpf_dma_mem *dma_mem; - int err; - - q_msg = kzalloc_objs(struct idpf_ctlq_msg *, num_q_msg, GFP_ATOMIC); - if (!q_msg) - return -ENOMEM; - - err = idpf_ctlq_clean_sq(asq, &num_q_msg, q_msg); - if (err) - goto err_kfree; - - for (i = 0; i < num_q_msg; i++) { - if (!q_msg[i]) - continue; - dma_mem = q_msg[i]->ctx.indirect.payload; - if (dma_mem) - dma_free_coherent(&adapter->pdev->dev, dma_mem->size, - dma_mem->va, dma_mem->pa); - kfree(q_msg[i]); - kfree(dma_mem); - } - -err_kfree: - kfree(q_msg); - - return err; + libie_ctlq_xn_send_clean(asq, kfree, deinit); } #if IS_ENABLED(CONFIG_PTP_1588_CLOCK) @@ -188,7 +158,7 @@ static bool idpf_ptp_is_mb_msg(u32 op) * @ctlq_msg: Corresponding control queue message */ static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op, - struct idpf_ctlq_msg *ctlq_msg) + struct libie_ctlq_msg *ctlq_msg) { /* If the message is PTP-related and the secondary mailbox is available, * send the message through the secondary mailbox. @@ -196,532 +166,108 @@ static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op, if (!idpf_ptp_is_mb_msg(op) || !adapter->ptp->secondary_mbx.valid) return; - ctlq_msg->opcode = idpf_mbq_opc_send_msg_to_peer_drv; + ctlq_msg->opcode = LIBIE_CTLQ_SEND_MSG_TO_PEER; ctlq_msg->func_id = adapter->ptp->secondary_mbx.peer_mbx_q_id; - ctlq_msg->host_id = adapter->ptp->secondary_mbx.peer_id; + ctlq_msg->flags = FIELD_PREP(LIBIE_CTLQ_DESC_FLAG_HOST_ID, + adapter->ptp->secondary_mbx.peer_id); } #else /* !CONFIG_PTP_1588_CLOCK */ static void idpf_prepare_ptp_mb_msg(struct idpf_adapter *adapter, u32 op, - struct idpf_ctlq_msg *ctlq_msg) + struct libie_ctlq_msg *ctlq_msg) { } #endif /* CONFIG_PTP_1588_CLOCK */ /** - * idpf_send_mb_msg - Send message over mailbox + * idpf_send_mb_msg - send mailbox message to the device control plane * @adapter: driver specific private structure - * @asq: control queue to send message to - * @op: virtchnl opcode - * @msg_size: size of the payload - * @msg: pointer to buffer holding the payload - * @cookie: unique SW generated cookie per message + * @xn_params: Xn send parameters to fill + * @send_buf: buffer to send + * @send_buf_size: size of the send buffer * - * Will prepare the control queue message and initiates the send api + * Fill the Xn parameters with the required info to send a virtchnl message. + * The send buffer is DMA mapped in the libie to avoid memcpy. * - * Return: 0 on success, negative on failure - */ -int idpf_send_mb_msg(struct idpf_adapter *adapter, struct idpf_ctlq_info *asq, - u32 op, u16 msg_size, u8 *msg, u16 cookie) -{ - struct idpf_ctlq_msg *ctlq_msg; - struct idpf_dma_mem *dma_mem; - int err; - - /* If we are here and a reset is detected nothing much can be - * done. This thread should silently abort and expected to - * be corrected with a new run either by user or driver - * flows after reset - */ - if (idpf_is_reset_detected(adapter)) - return 0; - - err = idpf_mb_clean(adapter, asq); - if (err) - return err; - - ctlq_msg = kzalloc_obj(*ctlq_msg, GFP_ATOMIC); - if (!ctlq_msg) - return -ENOMEM; - - dma_mem = kzalloc_obj(*dma_mem, GFP_ATOMIC); - if (!dma_mem) { - err = -ENOMEM; - goto dma_mem_error; - } - - ctlq_msg->opcode = idpf_mbq_opc_send_msg_to_cp; - ctlq_msg->func_id = 0; - - idpf_prepare_ptp_mb_msg(adapter, op, ctlq_msg); - - ctlq_msg->data_len = msg_size; - ctlq_msg->cookie.mbx.chnl_opcode = op; - ctlq_msg->cookie.mbx.chnl_retval = 0; - dma_mem->size = IDPF_CTLQ_MAX_BUF_LEN; - dma_mem->va = dma_alloc_coherent(&adapter->pdev->dev, dma_mem->size, - &dma_mem->pa, GFP_ATOMIC); - if (!dma_mem->va) { - err = -ENOMEM; - goto dma_alloc_error; - } - - /* It's possible we're just sending an opcode but no buffer */ - if (msg && msg_size) - memcpy(dma_mem->va, msg, msg_size); - ctlq_msg->ctx.indirect.payload = dma_mem; - ctlq_msg->ctx.sw_cookie.data = cookie; - - err = idpf_ctlq_send(&adapter->hw, asq, 1, ctlq_msg); - if (err) - goto send_error; - - return 0; - -send_error: - dma_free_coherent(&adapter->pdev->dev, dma_mem->size, dma_mem->va, - dma_mem->pa); -dma_alloc_error: - kfree(dma_mem); -dma_mem_error: - kfree(ctlq_msg); - - return err; -} - -/* API for virtchnl "transaction" support ("xn" for short). */ - -/** - * idpf_vc_xn_lock - Request exclusive access to vc transaction - * @xn: struct idpf_vc_xn* to access - */ -#define idpf_vc_xn_lock(xn) \ - spin_lock(&(xn)->lock) - -/** - * idpf_vc_xn_unlock - Release exclusive access to vc transaction - * @xn: struct idpf_vc_xn* to access - */ -#define idpf_vc_xn_unlock(xn) \ - spin_unlock(&(xn)->lock) - -/** - * idpf_vc_xn_release_bufs - Release reference to reply buffer(s) and - * reset the transaction state. - * @xn: struct idpf_vc_xn to update - */ -static void idpf_vc_xn_release_bufs(struct idpf_vc_xn *xn) -{ - xn->reply.iov_base = NULL; - xn->reply.iov_len = 0; - - if (xn->state != IDPF_VC_XN_SHUTDOWN) - xn->state = IDPF_VC_XN_IDLE; -} - -/** - * idpf_vc_xn_init - Initialize virtchnl transaction object - * @vcxn_mngr: pointer to vc transaction manager struct - */ -static void idpf_vc_xn_init(struct idpf_vc_xn_manager *vcxn_mngr) -{ - int i; - - spin_lock_init(&vcxn_mngr->xn_bm_lock); - - for (i = 0; i < ARRAY_SIZE(vcxn_mngr->ring); i++) { - struct idpf_vc_xn *xn = &vcxn_mngr->ring[i]; - - xn->state = IDPF_VC_XN_IDLE; - xn->idx = i; - idpf_vc_xn_release_bufs(xn); - spin_lock_init(&xn->lock); - init_completion(&xn->completed); - } - - bitmap_fill(vcxn_mngr->free_xn_bm, IDPF_VC_XN_RING_LEN); -} - -/** - * idpf_vc_xn_shutdown - Uninitialize virtchnl transaction object - * @vcxn_mngr: pointer to vc transaction manager struct + * Cleanup the mailbox queue entries of the previously sent message to + * unmap and release the buffer. * - * All waiting threads will be woken-up and their transaction aborted. Further - * operations on that object will fail. + * Return: 0 if the request was successful, -%EBUSY if reset is detected + * or Tx control queue is full, other negative error code on failure. */ -void idpf_vc_xn_shutdown(struct idpf_vc_xn_manager *vcxn_mngr) +int idpf_send_mb_msg(struct idpf_adapter *adapter, + struct libie_ctlq_xn_send_params *xn_params, + void *send_buf, size_t send_buf_size) { - int i; + struct libie_ctlq_msg ctlq_msg = {}; - spin_lock_bh(&vcxn_mngr->xn_bm_lock); - bitmap_zero(vcxn_mngr->free_xn_bm, IDPF_VC_XN_RING_LEN); - spin_unlock_bh(&vcxn_mngr->xn_bm_lock); + if (idpf_is_reset_detected(adapter)) { + if (!libie_cp_can_send_onstack(send_buf_size)) + kfree(send_buf); - for (i = 0; i < ARRAY_SIZE(vcxn_mngr->ring); i++) { - struct idpf_vc_xn *xn = &vcxn_mngr->ring[i]; - - idpf_vc_xn_lock(xn); - xn->state = IDPF_VC_XN_SHUTDOWN; - idpf_vc_xn_release_bufs(xn); - idpf_vc_xn_unlock(xn); - complete_all(&xn->completed); + return -EBUSY; } + + idpf_prepare_ptp_mb_msg(adapter, xn_params->chnl_opcode, &ctlq_msg); + xn_params->ctlq_msg = ctlq_msg.opcode ? &ctlq_msg : NULL; + + xn_params->send_buf.iov_base = send_buf; + xn_params->send_buf.iov_len = send_buf_size; + xn_params->xnm = adapter->xnm; + xn_params->ctlq = xn_params->ctlq ? xn_params->ctlq : adapter->asq; + xn_params->rel_tx_buf = kfree; + + idpf_mb_clean(xn_params->ctlq, false); + + return libie_ctlq_xn_send(xn_params); } /** - * idpf_vc_xn_pop_free - Pop a free transaction from free list - * @vcxn_mngr: transaction manager to pop from - * - * Returns NULL if no free transactions - */ -static -struct idpf_vc_xn *idpf_vc_xn_pop_free(struct idpf_vc_xn_manager *vcxn_mngr) -{ - struct idpf_vc_xn *xn = NULL; - unsigned long free_idx; - - spin_lock_bh(&vcxn_mngr->xn_bm_lock); - free_idx = find_first_bit(vcxn_mngr->free_xn_bm, IDPF_VC_XN_RING_LEN); - if (free_idx == IDPF_VC_XN_RING_LEN) - goto do_unlock; - - clear_bit(free_idx, vcxn_mngr->free_xn_bm); - xn = &vcxn_mngr->ring[free_idx]; - xn->salt = vcxn_mngr->salt++; - -do_unlock: - spin_unlock_bh(&vcxn_mngr->xn_bm_lock); - - return xn; -} - -/** - * idpf_vc_xn_push_free - Push a free transaction to free list - * @vcxn_mngr: transaction manager to push to - * @xn: transaction to push - */ -static void idpf_vc_xn_push_free(struct idpf_vc_xn_manager *vcxn_mngr, - struct idpf_vc_xn *xn) -{ - idpf_vc_xn_release_bufs(xn); - spin_lock_bh(&vcxn_mngr->xn_bm_lock); - set_bit(xn->idx, vcxn_mngr->free_xn_bm); - spin_unlock_bh(&vcxn_mngr->xn_bm_lock); -} - -/** - * idpf_vc_xn_exec - Perform a send/recv virtchnl transaction - * @adapter: driver specific private structure with vcxn_mngr - * @params: parameters for this particular transaction including - * -vc_op: virtchannel operation to send - * -send_buf: kvec iov for send buf and len - * -recv_buf: kvec iov for recv buf and len (ignored if NULL) - * -timeout_ms: timeout waiting for a reply (milliseconds) - * -async: don't wait for message reply, will lose caller context - * -async_handler: callback to handle async replies - * - * @returns >= 0 for success, the size of the initial reply (may or may not be - * >= @recv_buf.iov_len, but we never overflow @@recv_buf_iov_base). < 0 for - * error. - */ -ssize_t idpf_vc_xn_exec(struct idpf_adapter *adapter, - const struct idpf_vc_xn_params *params) -{ - const struct kvec *send_buf = ¶ms->send_buf; - struct idpf_vc_xn *xn; - ssize_t retval; - u16 cookie; - - xn = idpf_vc_xn_pop_free(adapter->vcxn_mngr); - /* no free transactions available */ - if (!xn) - return -ENOSPC; - - idpf_vc_xn_lock(xn); - if (xn->state == IDPF_VC_XN_SHUTDOWN) { - retval = -ENXIO; - goto only_unlock; - } else if (xn->state != IDPF_VC_XN_IDLE) { - /* We're just going to clobber this transaction even though - * it's not IDLE. If we don't reuse it we could theoretically - * eventually leak all the free transactions and not be able to - * send any messages. At least this way we make an attempt to - * remain functional even though something really bad is - * happening that's corrupting what was supposed to be free - * transactions. - */ - WARN_ONCE(1, "There should only be idle transactions in free list (idx %d op %d)\n", - xn->idx, xn->vc_op); - } - - xn->reply = params->recv_buf; - xn->reply_sz = 0; - xn->state = params->async ? IDPF_VC_XN_ASYNC : IDPF_VC_XN_WAITING; - xn->vc_op = params->vc_op; - xn->async_handler = params->async_handler; - idpf_vc_xn_unlock(xn); - - if (!params->async) - reinit_completion(&xn->completed); - cookie = FIELD_PREP(IDPF_VC_XN_SALT_M, xn->salt) | - FIELD_PREP(IDPF_VC_XN_IDX_M, xn->idx); - - retval = idpf_send_mb_msg(adapter, adapter->hw.asq, params->vc_op, - send_buf->iov_len, send_buf->iov_base, - cookie); - if (retval) { - idpf_vc_xn_lock(xn); - goto release_and_unlock; - } - - if (params->async) - return 0; - - wait_for_completion_timeout(&xn->completed, - msecs_to_jiffies(params->timeout_ms)); - - /* No need to check the return value; we check the final state of the - * transaction below. It's possible the transaction actually gets more - * timeout than specified if we get preempted here but after - * wait_for_completion_timeout returns. This should be non-issue - * however. - */ - idpf_vc_xn_lock(xn); - switch (xn->state) { - case IDPF_VC_XN_SHUTDOWN: - retval = -ENXIO; - goto only_unlock; - case IDPF_VC_XN_WAITING: - dev_notice_ratelimited(&adapter->pdev->dev, - "Transaction timed-out (op:%d cookie:%04x vc_op:%d salt:%02x timeout:%dms)\n", - params->vc_op, cookie, xn->vc_op, - xn->salt, params->timeout_ms); - retval = -ETIME; - break; - case IDPF_VC_XN_COMPLETED_SUCCESS: - retval = xn->reply_sz; - break; - case IDPF_VC_XN_COMPLETED_FAILED: - dev_notice_ratelimited(&adapter->pdev->dev, "Transaction failed (op %d)\n", - params->vc_op); - retval = -EIO; - break; - default: - /* Invalid state. */ - WARN_ON_ONCE(1); - retval = -EIO; - break; - } - -release_and_unlock: - idpf_vc_xn_push_free(adapter->vcxn_mngr, xn); - /* If we receive a VC reply after here, it will be dropped. */ -only_unlock: - idpf_vc_xn_unlock(xn); - - return retval; -} - -/** - * idpf_vc_xn_forward_async - Handle async reply receives - * @adapter: private data struct - * @xn: transaction to handle - * @ctlq_msg: corresponding ctlq_msg - * - * For async sends we're going to lose the caller's context so, if an - * async_handler was provided, it can deal with the reply, otherwise we'll just - * check and report if there is an error. - */ -static int -idpf_vc_xn_forward_async(struct idpf_adapter *adapter, struct idpf_vc_xn *xn, - const struct idpf_ctlq_msg *ctlq_msg) -{ - int err = 0; - - if (ctlq_msg->cookie.mbx.chnl_opcode != xn->vc_op) { - dev_err_ratelimited(&adapter->pdev->dev, "Async message opcode does not match transaction opcode (msg: %d) (xn: %d)\n", - ctlq_msg->cookie.mbx.chnl_opcode, xn->vc_op); - xn->reply_sz = 0; - err = -EINVAL; - goto release_bufs; - } - - if (xn->async_handler) { - err = xn->async_handler(adapter, xn, ctlq_msg); - goto release_bufs; - } - - if (ctlq_msg->cookie.mbx.chnl_retval) { - xn->reply_sz = 0; - dev_err_ratelimited(&adapter->pdev->dev, "Async message failure (op %d)\n", - ctlq_msg->cookie.mbx.chnl_opcode); - err = -EINVAL; - } - -release_bufs: - idpf_vc_xn_push_free(adapter->vcxn_mngr, xn); - - return err; -} - -/** - * idpf_vc_xn_forward_reply - copy a reply back to receiving thread - * @adapter: driver specific private structure with vcxn_mngr - * @ctlq_msg: controlq message to send back to receiving thread - */ -static int -idpf_vc_xn_forward_reply(struct idpf_adapter *adapter, - const struct idpf_ctlq_msg *ctlq_msg) -{ - const void *payload = NULL; - size_t payload_size = 0; - struct idpf_vc_xn *xn; - u16 msg_info; - int err = 0; - u16 xn_idx; - u16 salt; - - msg_info = ctlq_msg->ctx.sw_cookie.data; - xn_idx = FIELD_GET(IDPF_VC_XN_IDX_M, msg_info); - if (xn_idx >= ARRAY_SIZE(adapter->vcxn_mngr->ring)) { - dev_err_ratelimited(&adapter->pdev->dev, "Out of bounds cookie received: %02x\n", - xn_idx); - return -EINVAL; - } - xn = &adapter->vcxn_mngr->ring[xn_idx]; - idpf_vc_xn_lock(xn); - salt = FIELD_GET(IDPF_VC_XN_SALT_M, msg_info); - if (xn->salt != salt) { - dev_err_ratelimited(&adapter->pdev->dev, "Transaction salt does not match (exp:%d@%02x(%d) != got:%d@%02x)\n", - xn->vc_op, xn->salt, xn->state, - ctlq_msg->cookie.mbx.chnl_opcode, salt); - idpf_vc_xn_unlock(xn); - return -EINVAL; - } - - switch (xn->state) { - case IDPF_VC_XN_WAITING: - /* success */ - break; - case IDPF_VC_XN_IDLE: - dev_err_ratelimited(&adapter->pdev->dev, "Unexpected or belated VC reply (op %d)\n", - ctlq_msg->cookie.mbx.chnl_opcode); - err = -EINVAL; - goto out_unlock; - case IDPF_VC_XN_SHUTDOWN: - /* ENXIO is a bit special here as the recv msg loop uses that - * know if it should stop trying to clean the ring if we lost - * the virtchnl. We need to stop playing with registers and - * yield. - */ - err = -ENXIO; - goto out_unlock; - case IDPF_VC_XN_ASYNC: - /* Set reply_sz from the actual payload so that async_handler - * can evaluate the response. - */ - xn->reply_sz = ctlq_msg->data_len; - err = idpf_vc_xn_forward_async(adapter, xn, ctlq_msg); - idpf_vc_xn_unlock(xn); - return err; - default: - dev_err_ratelimited(&adapter->pdev->dev, "Overwriting VC reply (op %d)\n", - ctlq_msg->cookie.mbx.chnl_opcode); - err = -EBUSY; - goto out_unlock; - } - - if (ctlq_msg->cookie.mbx.chnl_opcode != xn->vc_op) { - dev_err_ratelimited(&adapter->pdev->dev, "Message opcode does not match transaction opcode (msg: %d) (xn: %d)\n", - ctlq_msg->cookie.mbx.chnl_opcode, xn->vc_op); - xn->reply_sz = 0; - xn->state = IDPF_VC_XN_COMPLETED_FAILED; - err = -EINVAL; - goto out_unlock; - } - - if (ctlq_msg->cookie.mbx.chnl_retval) { - xn->reply_sz = 0; - xn->state = IDPF_VC_XN_COMPLETED_FAILED; - err = -EINVAL; - goto out_unlock; - } - - if (ctlq_msg->data_len) { - payload = ctlq_msg->ctx.indirect.payload->va; - payload_size = ctlq_msg->data_len; - } - - xn->reply_sz = payload_size; - xn->state = IDPF_VC_XN_COMPLETED_SUCCESS; - - if (xn->reply.iov_base && xn->reply.iov_len && payload_size) - memcpy(xn->reply.iov_base, payload, - min_t(size_t, xn->reply.iov_len, payload_size)); - -out_unlock: - idpf_vc_xn_unlock(xn); - /* we _cannot_ hold lock while calling complete */ - complete(&xn->completed); - - return err; -} - -/** - * idpf_recv_mb_msg - Receive message over mailbox + * idpf_send_mb_msg_kfree - send mailbox message and free the send buffer * @adapter: driver specific private structure - * @arq: control queue to receive message from + * @xn_params: Xn send parameters to fill + * @send_buf: buffer to send, can be released with kfree() + * @send_buf_size: size of the send buffer * - * Will receive control queue message and posts the receive buffer. + * libie_cp functions consume only buffers above certain size, + * smaller buffers are assumed to be on the stack. However, for some + * commands with variable message size it makes sense to always use kzalloc(), + * which means we have to free smaller buffers ourselves. * - * Return: 0 on success and negative on failure. + * Return: 0 if no unexpected errors were encountered, + * negative error code otherwise. */ -int idpf_recv_mb_msg(struct idpf_adapter *adapter, struct idpf_ctlq_info *arq) +int idpf_send_mb_msg_kfree(struct idpf_adapter *adapter, + struct libie_ctlq_xn_send_params *xn_params, + void *send_buf, size_t send_buf_size) { - struct idpf_ctlq_msg ctlq_msg; - struct idpf_dma_mem *dma_mem; - int post_err, err; - u16 num_recv; + int err = idpf_send_mb_msg(adapter, xn_params, send_buf, send_buf_size); - while (1) { - /* This will get <= num_recv messages and output how many - * actually received on num_recv. - */ - num_recv = 1; - err = idpf_ctlq_recv(arq, &num_recv, &ctlq_msg); - if (err || !num_recv) - break; - - if (ctlq_msg.data_len) { - dma_mem = ctlq_msg.ctx.indirect.payload; - } else { - dma_mem = NULL; - num_recv = 0; - } - - if (ctlq_msg.cookie.mbx.chnl_opcode == VIRTCHNL2_OP_EVENT) - idpf_recv_event_msg(adapter, &ctlq_msg); - else - err = idpf_vc_xn_forward_reply(adapter, &ctlq_msg); - - post_err = idpf_ctlq_post_rx_buffs(&adapter->hw, arq, - &num_recv, &dma_mem); - - /* If post failed clear the only buffer we supplied */ - if (post_err) { - if (dma_mem) - dma_free_coherent(&adapter->pdev->dev, - dma_mem->size, dma_mem->va, - dma_mem->pa); - break; - } - - /* virtchnl trying to shutdown, stop cleaning */ - if (err == -ENXIO) - break; - } + if (libie_cp_can_send_onstack(send_buf_size)) + kfree(send_buf); return err; } +/** + * idpf_send_vf_reset_msg - send one way VF reset message + * @adapter: driver specific private structure + */ +void idpf_send_vf_reset_msg(struct idpf_adapter *adapter) +{ + struct libie_ctlq_info *ctlq = adapter->asq; + + /* Forcefully claim send queue slot */ + idpf_mb_clean(ctlq, true); + + scoped_guard(spinlock, &ctlq->lock) { + *ctlq->tx_msg[ctlq->next_to_use] = (struct libie_ctlq_msg) { + .opcode = LIBIE_CTLQ_SEND_MSG_TO_CP, + .chnl_opcode = VIRTCHNL2_OP_RESET_VF, + }; + + libie_ctlq_send(adapter->asq, 1); + } +} + struct idpf_chunked_msg_params { u32 (*prepare_msg)(u32 vport_id, void *buf, const void *pos, u32 num); @@ -767,45 +313,43 @@ struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_adapter *adapter, static int idpf_send_chunked_msg(struct idpf_adapter *adapter, const struct idpf_chunked_msg_params *params) { - struct idpf_vc_xn_params xn_params = { - .vc_op = params->vc_op, + struct libie_ctlq_xn_send_params xn_params = { .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = params->vc_op, }; const void *pos = params->chunks; - u32 num_chunks, num_msgs, buf_sz; - void *buf __free(kfree) = NULL; u32 totqs = params->num_chunks; u32 vid = params->vport_id; + u32 num_chunks, num_msgs; - num_chunks = min(IDPF_NUM_CHUNKS_PER_MSG(params->config_sz, - params->chunk_sz), totqs); + num_chunks = IDPF_NUM_CHUNKS_PER_MSG(params->config_sz, + params->chunk_sz); num_msgs = DIV_ROUND_UP(totqs, num_chunks); - buf_sz = params->config_sz + num_chunks * params->chunk_sz; - buf = kzalloc(buf_sz, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - xn_params.send_buf.iov_base = buf; - for (u32 i = 0; i < num_msgs; i++) { - ssize_t reply_sz; - - memset(buf, 0, buf_sz); - xn_params.send_buf.iov_len = buf_sz; - - if (params->prepare_msg(vid, buf, pos, num_chunks) != buf_sz) - return -EINVAL; - - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - - pos += num_chunks * params->chunk_sz; - totqs -= num_chunks; + u32 buf_sz; + void *buf; + int err; num_chunks = min(num_chunks, totqs); buf_sz = params->config_sz + num_chunks * params->chunk_sz; + buf = kzalloc(buf_sz, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + if (params->prepare_msg(vid, buf, pos, num_chunks) != buf_sz) { + kfree(buf); + return -EINVAL; + } + + err = idpf_send_mb_msg_kfree(adapter, &xn_params, buf, buf_sz); + if (err) + return err; + + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + xn_params.recv_mem = (struct kvec) {}; + pos += num_chunks * params->chunk_sz; + totqs -= num_chunks; } return 0; @@ -880,11 +424,14 @@ static int idpf_wait_for_marker_event(struct idpf_vport *vport) */ static int idpf_send_ver_msg(struct idpf_adapter *adapter) { - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_VERSION, + }; + struct virtchnl2_version_info *vvi_recv; struct virtchnl2_version_info vvi; - ssize_t reply_sz; u32 major, minor; - int err = 0; + int err; if (adapter->virt_ver_maj) { vvi.major = cpu_to_le32(adapter->virt_ver_maj); @@ -894,24 +441,23 @@ static int idpf_send_ver_msg(struct idpf_adapter *adapter) vvi.minor = cpu_to_le32(IDPF_VIRTCHNL_VERSION_MINOR); } - xn_params.vc_op = VIRTCHNL2_OP_VERSION; - xn_params.send_buf.iov_base = &vvi; - xn_params.send_buf.iov_len = sizeof(vvi); - xn_params.recv_buf = xn_params.send_buf; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; + err = idpf_send_mb_msg_stack(adapter, &xn_params, &vvi); + if (err) + return err; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (reply_sz < sizeof(vvi)) - return -EIO; + if (xn_params.recv_mem.iov_len < sizeof(*vvi_recv)) { + err = -EIO; + goto free_rx_buf; + } - major = le32_to_cpu(vvi.major); - minor = le32_to_cpu(vvi.minor); + vvi_recv = xn_params.recv_mem.iov_base; + major = le32_to_cpu(vvi_recv->major); + minor = le32_to_cpu(vvi_recv->minor); if (major > IDPF_VIRTCHNL_VERSION_MAJOR) { dev_warn(&adapter->pdev->dev, "Virtchnl major version greater than supported\n"); - return -EINVAL; + err = -EINVAL; + goto free_rx_buf; } if (major == IDPF_VIRTCHNL_VERSION_MAJOR && @@ -929,6 +475,9 @@ static int idpf_send_ver_msg(struct idpf_adapter *adapter) adapter->virt_ver_maj = major; adapter->virt_ver_min = minor; +free_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + return err; } @@ -941,9 +490,12 @@ static int idpf_send_ver_msg(struct idpf_adapter *adapter) */ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter) { + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_GET_CAPS, + }; struct virtchnl2_get_capabilities caps = {}; - struct idpf_vc_xn_params xn_params = {}; - ssize_t reply_sz; + int err; caps.csum_caps = cpu_to_le32(VIRTCHNL2_CAP_TX_CSUM_L3_IPV4 | @@ -1003,145 +555,156 @@ static int idpf_send_get_caps_msg(struct idpf_adapter *adapter) VIRTCHNL2_CAP_LOOPBACK | VIRTCHNL2_CAP_PTP); - xn_params.vc_op = VIRTCHNL2_OP_GET_CAPS; - xn_params.send_buf.iov_base = ∩︀ - xn_params.send_buf.iov_len = sizeof(caps); - xn_params.recv_buf.iov_base = &adapter->caps; - xn_params.recv_buf.iov_len = sizeof(adapter->caps); - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; + err = idpf_send_mb_msg_stack(adapter, &xn_params, &caps); + if (err) + return err; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (reply_sz < sizeof(adapter->caps)) - return -EIO; - - return 0; -} - -/** - * idpf_send_get_lan_memory_regions - Send virtchnl get LAN memory regions msg - * @adapter: Driver specific private struct - * - * Return: 0 on success or error code on failure. - */ -static int idpf_send_get_lan_memory_regions(struct idpf_adapter *adapter) -{ - struct virtchnl2_get_lan_memory_regions *rcvd_regions __free(kfree); - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_GET_LAN_MEMORY_REGIONS, - .recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN, - .send_buf.iov_len = - sizeof(struct virtchnl2_get_lan_memory_regions) + - sizeof(struct virtchnl2_mem_region), - .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, - }; - int num_regions, size; - struct idpf_hw *hw; - ssize_t reply_sz; - int err = 0; - - rcvd_regions = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL); - if (!rcvd_regions) - return -ENOMEM; - - xn_params.recv_buf.iov_base = rcvd_regions; - rcvd_regions->num_memory_regions = cpu_to_le16(1); - xn_params.send_buf.iov_base = rcvd_regions; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - - num_regions = le16_to_cpu(rcvd_regions->num_memory_regions); - size = struct_size(rcvd_regions, mem_reg, num_regions); - if (reply_sz < size) - return -EIO; - - if (size > IDPF_CTLQ_MAX_BUF_LEN) - return -EINVAL; - - hw = &adapter->hw; - hw->lan_regs = kzalloc_objs(*hw->lan_regs, num_regions); - if (!hw->lan_regs) - return -ENOMEM; - - for (int i = 0; i < num_regions; i++) { - hw->lan_regs[i].addr_len = - le64_to_cpu(rcvd_regions->mem_reg[i].size); - hw->lan_regs[i].addr_start = - le64_to_cpu(rcvd_regions->mem_reg[i].start_offset); + if (xn_params.recv_mem.iov_len < sizeof(adapter->caps)) { + err = -EIO; + goto free_rx_buf; } - hw->num_lan_regs = num_regions; + + memcpy(&adapter->caps, xn_params.recv_mem.iov_base, + sizeof(adapter->caps)); + +free_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); return err; } /** - * idpf_calc_remaining_mmio_regs - calculate MMIO regions outside mbx and rstat - * @adapter: Driver specific private structure + * idpf_mmio_region_non_static - Check if region is not static + * @mmio_info: PCI resources info + * @reg: region to check * - * Called when idpf_send_get_lan_memory_regions is not supported. This will - * calculate the offsets and sizes for the regions before, in between, and - * after the mailbox and rstat MMIO mappings. - * - * Return: 0 on success or error code on failure. + * Return: %true if region can be received though virtchnl command, + * %false if region is related to mailbox or resetting */ -static int idpf_calc_remaining_mmio_regs(struct idpf_adapter *adapter) +bool idpf_mmio_region_non_static(struct libie_mmio_info *mmio_info, + struct libie_pci_mmio_region *reg) { - struct resource *rstat_reg = &adapter->dev_ops.static_reg_info[1]; - struct resource *mbx_reg = &adapter->dev_ops.static_reg_info[0]; - struct idpf_hw *hw = &adapter->hw; + struct idpf_adapter *adapter = + container_of(mmio_info, struct idpf_adapter, + ctlq_ctx.mmio_info); - hw->num_lan_regs = IDPF_MMIO_MAP_FALLBACK_MAX_REMAINING; - hw->lan_regs = kzalloc_objs(*hw->lan_regs, hw->num_lan_regs); - if (!hw->lan_regs) - return -ENOMEM; + for (uint i = 0; i < IDPF_MMIO_REG_NUM_STATIC; i++) { + if (reg->bar_idx == 0 && + reg->offset == adapter->dev_ops.static_reg_info[i].start) + return false; + } - /* Region preceding mailbox */ - hw->lan_regs[0].addr_start = 0; - hw->lan_regs[0].addr_len = mbx_reg->start; - /* Region between mailbox and rstat */ - hw->lan_regs[1].addr_start = mbx_reg->end + 1; - hw->lan_regs[1].addr_len = rstat_reg->start - - hw->lan_regs[1].addr_start; - /* Region after rstat */ - hw->lan_regs[2].addr_start = rstat_reg->end + 1; - hw->lan_regs[2].addr_len = pci_resource_len(adapter->pdev, 0) - - hw->lan_regs[2].addr_start; - - return 0; + return true; } /** - * idpf_map_lan_mmio_regs - map remaining LAN BAR regions + * idpf_decfg_lan_memory_regions - Unmap non-static memory regions * @adapter: Driver specific private structure + */ +static void idpf_decfg_lan_memory_regions(struct idpf_adapter *adapter) +{ + libie_pci_unmap_fltr_regs(&adapter->ctlq_ctx.mmio_info, + idpf_mmio_region_non_static); +} + +/** + * idpf_cfg_lan_memory_regions - Get (via virtchnl) and map LAN memory regions + * @adapter: Driver specific private struct * * Return: 0 on success or error code on failure. */ -static int idpf_map_lan_mmio_regs(struct idpf_adapter *adapter) +static int idpf_cfg_lan_memory_regions(struct idpf_adapter *adapter) { - struct pci_dev *pdev = adapter->pdev; - struct idpf_hw *hw = &adapter->hw; - resource_size_t res_start; + struct virtchnl2_get_lan_memory_regions *send_regions, *rcvd_regions; + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_GET_LAN_MEMORY_REGIONS, + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + }; + size_t send_sz, reply_sz, size; + int num_regions; + int err = 0; - res_start = pci_resource_start(pdev, 0); + send_sz = sizeof(struct virtchnl2_get_lan_memory_regions) + + sizeof(struct virtchnl2_mem_region); + send_regions = kzalloc(send_sz, GFP_KERNEL); + if (!send_regions) + return -ENOMEM; - for (int i = 0; i < hw->num_lan_regs; i++) { - resource_size_t start; - long len; + send_regions->num_memory_regions = cpu_to_le16(1); + err = idpf_send_mb_msg_kfree(adapter, &xn_params, send_regions, + send_sz); + if (err) + return err; - len = hw->lan_regs[i].addr_len; - if (!len) - continue; - start = hw->lan_regs[i].addr_start + res_start; + rcvd_regions = xn_params.recv_mem.iov_base; + reply_sz = xn_params.recv_mem.iov_len; + if (reply_sz < sizeof(*rcvd_regions)) { + err = -EIO; + goto rel_rx_buf; + } + num_regions = le16_to_cpu(rcvd_regions->num_memory_regions); + size = struct_size(rcvd_regions, mem_reg, num_regions); + if (reply_sz < size) { + err = -EIO; + goto rel_rx_buf; + } - hw->lan_regs[i].vaddr = devm_ioremap(&pdev->dev, start, len); - if (!hw->lan_regs[i].vaddr) { - pci_err(pdev, "failed to allocate BAR0 region\n"); - return -ENOMEM; + for (int i = 0; i < num_regions; i++) { + struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; + resource_size_t offset, len; + + offset = le64_to_cpu(rcvd_regions->mem_reg[i].start_offset); + len = le64_to_cpu(rcvd_regions->mem_reg[i].size); + if (len && !libie_pci_map_mmio_region(mmio, offset, len)) { + idpf_decfg_lan_memory_regions(adapter); + err = -EIO; + goto rel_rx_buf; } } +rel_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return err; +} + +/** + * idpf_map_remaining_mmio_regs - map MMIO regions outside mbx and rstat + * @adapter: Driver specific private structure + * + * Called when idpf_cfg_lan_memory_regions is not supported. This will + * calculate the offsets and sizes for the regions before, in between, and + * after the mailbox and rstat MMIO mappings, and map those ranges. + * + * Return: 0 on success or error code on failure. + */ +static int idpf_map_remaining_mmio_regs(struct idpf_adapter *adapter) +{ + struct resource *rstat_reg = &adapter->dev_ops.static_reg_info[1]; + struct resource *mbx_reg = &adapter->dev_ops.static_reg_info[0]; + struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; + resource_size_t reg_start, size; + bool ok = true; + + /* Region preceding mailbox */ + size = mbx_reg->start; + ok &= !size || libie_pci_map_mmio_region(mmio, 0, size); + + /* Region between mailbox and rstat */ + reg_start = mbx_reg->end + 1; + size = rstat_reg->start - reg_start; + ok &= !size || libie_pci_map_mmio_region(mmio, reg_start, size); + + /* Region after rstat */ + reg_start = rstat_reg->end + 1; + size = pci_resource_len(adapter->pdev, 0) - reg_start; + ok &= !size || libie_pci_map_mmio_region(mmio, reg_start, size); + + if (!ok) { + idpf_decfg_lan_memory_regions(adapter); + return -ENOMEM; + } + return 0; } @@ -1160,24 +723,43 @@ int idpf_add_del_fsteer_filters(struct idpf_adapter *adapter, struct virtchnl2_flow_rule_add_del *rule, enum virtchnl2_op opcode) { + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = opcode, + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + }; + struct virtchnl2_flow_rule_add_del *rx_rule; int rule_count = le32_to_cpu(rule->count); - struct idpf_vc_xn_params xn_params = {}; - ssize_t reply_sz; + size_t send_sz; + int err; if (opcode != VIRTCHNL2_OP_ADD_FLOW_RULE && - opcode != VIRTCHNL2_OP_DEL_FLOW_RULE) + opcode != VIRTCHNL2_OP_DEL_FLOW_RULE) { + kfree(rule); return -EINVAL; + } - xn_params.vc_op = opcode; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.async = false; - xn_params.send_buf.iov_base = rule; - xn_params.send_buf.iov_len = struct_size(rule, rule_info, rule_count); - xn_params.recv_buf.iov_base = rule; - xn_params.recv_buf.iov_len = struct_size(rule, rule_info, rule_count); + send_sz = struct_size(rule, rule_info, rule_count); + err = idpf_send_mb_msg_kfree(adapter, &xn_params, rule, send_sz); + if (err) + return err; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - return reply_sz < 0 ? reply_sz : 0; + if (xn_params.recv_mem.iov_len < send_sz) { + err = -EIO; + goto rel_rx; + } + + rx_rule = xn_params.recv_mem.iov_base; + for (int i = 0; i < rule_count; i++) { + if (rx_rule->rule_info[i].status != + cpu_to_le32(VIRTCHNL2_FLOW_RULE_SUCCESS)) { + err = -EIO; + goto rel_rx; + } + } + +rel_rx: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + return err; } /** @@ -1414,7 +996,7 @@ static int __idpf_queue_reg_init(struct idpf_vport *vport, struct idpf_q_vec_rsrc *rsrc, u32 *reg_vals, int num_regs, u32 q_type) { - struct idpf_adapter *adapter = vport->adapter; + struct libie_mmio_info *mmio = &vport->adapter->ctlq_ctx.mmio_info; int i, j, k = 0; switch (q_type) { @@ -1424,7 +1006,8 @@ static int __idpf_queue_reg_init(struct idpf_vport *vport, for (j = 0; j < tx_qgrp->num_txq && k < num_regs; j++, k++) tx_qgrp->txqs[j]->tail = - idpf_get_reg_addr(adapter, reg_vals[k]); + libie_pci_get_mmio_addr(mmio, + reg_vals[k]); } break; case VIRTCHNL2_QUEUE_TYPE_RX: @@ -1436,8 +1019,8 @@ static int __idpf_queue_reg_init(struct idpf_vport *vport, struct idpf_rx_queue *q; q = rx_qgrp->singleq.rxqs[j]; - q->tail = idpf_get_reg_addr(adapter, - reg_vals[k]); + q->tail = libie_pci_get_mmio_addr(mmio, + reg_vals[k]); } } break; @@ -1450,8 +1033,8 @@ static int __idpf_queue_reg_init(struct idpf_vport *vport, struct idpf_buf_queue *q; q = &rx_qgrp->splitq.bufq_sets[j].bufq; - q->tail = idpf_get_reg_addr(adapter, - reg_vals[k]); + q->tail = libie_pci_get_mmio_addr(mmio, + reg_vals[k]); } } break; @@ -1551,21 +1134,19 @@ int idpf_queue_reg_init(struct idpf_vport *vport, int idpf_send_create_vport_msg(struct idpf_adapter *adapter, struct idpf_vport_max_q *max_q) { + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_CREATE_VPORT, + }; struct virtchnl2_create_vport *vport_msg; - struct idpf_vc_xn_params xn_params = {}; u16 idx = adapter->next_vport; int err, buf_size; - ssize_t reply_sz; buf_size = sizeof(struct virtchnl2_create_vport); - if (!adapter->vport_params_reqd[idx]) { - adapter->vport_params_reqd[idx] = kzalloc(buf_size, - GFP_KERNEL); - if (!adapter->vport_params_reqd[idx]) - return -ENOMEM; - } + vport_msg = kzalloc(buf_size, GFP_KERNEL); + if (!vport_msg) + return -ENOMEM; - vport_msg = adapter->vport_params_reqd[idx]; vport_msg->vport_type = cpu_to_le16(VIRTCHNL2_VPORT_TYPE_DEFAULT); vport_msg->vport_index = cpu_to_le16(idx); @@ -1582,38 +1163,35 @@ int idpf_send_create_vport_msg(struct idpf_adapter *adapter, err = idpf_vport_calc_total_qs(adapter, idx, vport_msg, max_q); if (err) { dev_err(&adapter->pdev->dev, "Enough queues are not available"); - - return err; + goto rel_buf; } if (!adapter->vport_params_recvd[idx]) { - adapter->vport_params_recvd[idx] = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, - GFP_KERNEL); + adapter->vport_params_recvd[idx] = + kzalloc(LIBIE_CTLQ_MAX_BUF_LEN, GFP_KERNEL); if (!adapter->vport_params_recvd[idx]) { err = -ENOMEM; - goto free_vport_params; + goto rel_buf; } } - xn_params.vc_op = VIRTCHNL2_OP_CREATE_VPORT; - xn_params.send_buf.iov_base = vport_msg; - xn_params.send_buf.iov_len = buf_size; - xn_params.recv_buf.iov_base = adapter->vport_params_recvd[idx]; - xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) { - err = reply_sz; - goto free_vport_params; + err = idpf_send_mb_msg_kfree(adapter, &xn_params, vport_msg, + sizeof(*vport_msg)); + if (err) { + kfree(adapter->vport_params_recvd[idx]); + adapter->vport_params_recvd[idx] = NULL; + return err; } + memcpy(adapter->vport_params_recvd[idx], xn_params.recv_mem.iov_base, + xn_params.recv_mem.iov_len); + + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + return 0; -free_vport_params: - kfree(adapter->vport_params_recvd[idx]); - adapter->vport_params_recvd[idx] = NULL; - kfree(adapter->vport_params_reqd[idx]); - adapter->vport_params_reqd[idx] = NULL; +rel_buf: + kfree(vport_msg); return err; } @@ -1673,19 +1251,22 @@ int idpf_check_supported_desc_ids(struct idpf_vport *vport) */ int idpf_send_destroy_vport_msg(struct idpf_adapter *adapter, u32 vport_id) { - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_DESTROY_VPORT, + }; struct virtchnl2_vport v_id; - ssize_t reply_sz; + int err; v_id.vport_id = cpu_to_le32(vport_id); - xn_params.vc_op = VIRTCHNL2_OP_DESTROY_VPORT; - xn_params.send_buf.iov_base = &v_id; - xn_params.send_buf.iov_len = sizeof(v_id); - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); + err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); + if (err) + return err; - return reply_sz < 0 ? reply_sz : 0; + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return 0; } /** @@ -1697,19 +1278,22 @@ int idpf_send_destroy_vport_msg(struct idpf_adapter *adapter, u32 vport_id) */ int idpf_send_enable_vport_msg(struct idpf_adapter *adapter, u32 vport_id) { - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_ENABLE_VPORT, + }; struct virtchnl2_vport v_id; - ssize_t reply_sz; + int err; v_id.vport_id = cpu_to_le32(vport_id); - xn_params.vc_op = VIRTCHNL2_OP_ENABLE_VPORT; - xn_params.send_buf.iov_base = &v_id; - xn_params.send_buf.iov_len = sizeof(v_id); - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); + err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); + if (err) + return err; - return reply_sz < 0 ? reply_sz : 0; + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return 0; } /** @@ -1721,19 +1305,22 @@ int idpf_send_enable_vport_msg(struct idpf_adapter *adapter, u32 vport_id) */ int idpf_send_disable_vport_msg(struct idpf_adapter *adapter, u32 vport_id) { - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_DISABLE_VPORT, + }; struct virtchnl2_vport v_id; - ssize_t reply_sz; + int err; v_id.vport_id = cpu_to_le32(vport_id); - xn_params.vc_op = VIRTCHNL2_OP_DISABLE_VPORT; - xn_params.send_buf.iov_base = &v_id; - xn_params.send_buf.iov_len = sizeof(v_id); - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); + err = idpf_send_mb_msg_stack(adapter, &xn_params, &v_id); + if (err) + return err; - return reply_sz < 0 ? reply_sz : 0; + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return 0; } /** @@ -2572,11 +2159,14 @@ int idpf_send_delete_queues_msg(struct idpf_adapter *adapter, struct idpf_queue_id_reg_info *chunks, u32 vport_id) { - struct virtchnl2_del_ena_dis_queues *eq __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - ssize_t reply_sz; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_DEL_QUEUES, + }; + struct virtchnl2_del_ena_dis_queues *eq; + ssize_t buf_size; u16 num_chunks; - int buf_size; + int err; num_chunks = chunks->num_chunks; buf_size = struct_size(eq, chunks.chunks, num_chunks); @@ -2591,13 +2181,13 @@ int idpf_send_delete_queues_msg(struct idpf_adapter *adapter, idpf_convert_reg_to_queue_chunks(eq->chunks.chunks, chunks->queue_chunks, num_chunks); - xn_params.vc_op = VIRTCHNL2_OP_DEL_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.send_buf.iov_base = eq; - xn_params.send_buf.iov_len = buf_size; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); + err = idpf_send_mb_msg_kfree(adapter, &xn_params, eq, buf_size); + if (err) + return err; - return reply_sz < 0 ? reply_sz : 0; + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return 0; } /** @@ -2635,15 +2225,14 @@ int idpf_send_add_queues_msg(struct idpf_adapter *adapter, struct idpf_q_vec_rsrc *rsrc, u32 vport_id) { - struct virtchnl2_add_queues *vc_msg __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_ADD_QUEUES, + }; + struct virtchnl2_add_queues *vc_msg; struct virtchnl2_add_queues aq = {}; - ssize_t reply_sz; - int size; - - vc_msg = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL); - if (!vc_msg) - return -ENOMEM; + size_t size; + int err; aq.vport_id = cpu_to_le32(vport_id); aq.num_tx_q = cpu_to_le16(rsrc->num_txq); @@ -2651,29 +2240,38 @@ int idpf_send_add_queues_msg(struct idpf_adapter *adapter, aq.num_rx_q = cpu_to_le16(rsrc->num_rxq); aq.num_rx_bufq = cpu_to_le16(rsrc->num_bufq); - xn_params.vc_op = VIRTCHNL2_OP_ADD_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.send_buf.iov_base = &aq; - xn_params.send_buf.iov_len = sizeof(aq); - xn_params.recv_buf.iov_base = vc_msg; - xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + err = idpf_send_mb_msg_stack(adapter, &xn_params, &aq); + if (err) + return err; + + vc_msg = xn_params.recv_mem.iov_base; + if (xn_params.recv_mem.iov_len < sizeof(*vc_msg)) { + err = -EIO; + goto free_rx_buf; + } /* compare vc_msg num queues with vport num queues */ if (le16_to_cpu(vc_msg->num_tx_q) != rsrc->num_txq || le16_to_cpu(vc_msg->num_rx_q) != rsrc->num_rxq || le16_to_cpu(vc_msg->num_tx_complq) != rsrc->num_complq || - le16_to_cpu(vc_msg->num_rx_bufq) != rsrc->num_bufq) - return -EINVAL; + le16_to_cpu(vc_msg->num_rx_bufq) != rsrc->num_bufq) { + err = -EINVAL; + goto free_rx_buf; + } size = struct_size(vc_msg, chunks.chunks, le16_to_cpu(vc_msg->chunks.num_chunks)); - if (reply_sz < size) - return -EIO; + if (xn_params.recv_mem.iov_len < size) { + err = -EIO; + goto free_rx_buf; + } - return idpf_vport_init_queue_reg_chunks(vport_config, &vc_msg->chunks); + err = idpf_vport_init_queue_reg_chunks(vport_config, &vc_msg->chunks); + +free_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return err; } /** @@ -2685,49 +2283,51 @@ int idpf_send_add_queues_msg(struct idpf_adapter *adapter, */ int idpf_send_alloc_vectors_msg(struct idpf_adapter *adapter, u16 num_vectors) { - struct virtchnl2_alloc_vectors *rcvd_vec __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_ALLOC_VECTORS, + }; + struct virtchnl2_alloc_vectors *rcvd_vec; struct virtchnl2_alloc_vectors ac = {}; - ssize_t reply_sz; u16 num_vchunks; - int size; + int size, err; ac.num_vectors = cpu_to_le16(num_vectors); - rcvd_vec = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL); - if (!rcvd_vec) - return -ENOMEM; + err = idpf_send_mb_msg_stack(adapter, &xn_params, &ac); + if (err) + return err; - xn_params.vc_op = VIRTCHNL2_OP_ALLOC_VECTORS; - xn_params.send_buf.iov_base = ∾ - xn_params.send_buf.iov_len = sizeof(ac); - xn_params.recv_buf.iov_base = rcvd_vec; - xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + rcvd_vec = xn_params.recv_mem.iov_base; + if (xn_params.recv_mem.iov_len < sizeof(*rcvd_vec)) { + err = -EIO; + goto free_rx_buf; + } num_vchunks = le16_to_cpu(rcvd_vec->vchunks.num_vchunks); size = struct_size(rcvd_vec, vchunks.vchunks, num_vchunks); - if (reply_sz < size) - return -EIO; - - if (size > IDPF_CTLQ_MAX_BUF_LEN) - return -EINVAL; + if (xn_params.recv_mem.iov_len < size) { + err = -EIO; + goto free_rx_buf; + } kfree(adapter->req_vec_chunks); adapter->req_vec_chunks = kmemdup(rcvd_vec, size, GFP_KERNEL); - if (!adapter->req_vec_chunks) - return -ENOMEM; + if (!adapter->req_vec_chunks) { + err = -ENOMEM; + goto free_rx_buf; + } if (le16_to_cpu(adapter->req_vec_chunks->num_vectors) < num_vectors) { kfree(adapter->req_vec_chunks); adapter->req_vec_chunks = NULL; - return -EINVAL; + err = -EINVAL; } - return 0; +free_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return err; } /** @@ -2739,24 +2339,28 @@ int idpf_send_alloc_vectors_msg(struct idpf_adapter *adapter, u16 num_vectors) int idpf_send_dealloc_vectors_msg(struct idpf_adapter *adapter) { struct virtchnl2_alloc_vectors *ac = adapter->req_vec_chunks; - struct virtchnl2_vector_chunks *vcs = &ac->vchunks; - struct idpf_vc_xn_params xn_params = {}; - ssize_t reply_sz; - int buf_size; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_DEALLOC_VECTORS, + }; + struct virtchnl2_vector_chunks *vcs; + int buf_size, err; - buf_size = struct_size(vcs, vchunks, le16_to_cpu(vcs->num_vchunks)); + buf_size = struct_size(&ac->vchunks, vchunks, + le16_to_cpu(ac->vchunks.num_vchunks)); + vcs = kmemdup(&ac->vchunks, buf_size, GFP_KERNEL); + if (!vcs) + return -ENOMEM; - xn_params.vc_op = VIRTCHNL2_OP_DEALLOC_VECTORS; - xn_params.send_buf.iov_base = vcs; - xn_params.send_buf.iov_len = buf_size; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + err = idpf_send_mb_msg_kfree(adapter, &xn_params, vcs, buf_size); + if (err) + return err; kfree(adapter->req_vec_chunks); adapter->req_vec_chunks = NULL; + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + return 0; } @@ -2780,18 +2384,22 @@ static int idpf_get_max_vfs(struct idpf_adapter *adapter) */ int idpf_send_set_sriov_vfs_msg(struct idpf_adapter *adapter, u16 num_vfs) { + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_SET_SRIOV_VFS, + }; struct virtchnl2_sriov_vfs_info svi = {}; - struct idpf_vc_xn_params xn_params = {}; - ssize_t reply_sz; + int err; svi.num_vfs = cpu_to_le16(num_vfs); - xn_params.vc_op = VIRTCHNL2_OP_SET_SRIOV_VFS; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.send_buf.iov_base = &svi; - xn_params.send_buf.iov_len = sizeof(svi); - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - return reply_sz < 0 ? reply_sz : 0; + err = idpf_send_mb_msg_stack(adapter, &xn_params, &svi); + if (err) + return err; + + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return 0; } /** @@ -2804,10 +2412,14 @@ int idpf_send_set_sriov_vfs_msg(struct idpf_adapter *adapter, u16 num_vfs) int idpf_send_get_stats_msg(struct idpf_netdev_priv *np, struct idpf_port_stats *port_stats) { + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_GET_STATS, + }; struct rtnl_link_stats64 *netstats = &np->netstats; + struct virtchnl2_vport_stats *stats_recv; struct virtchnl2_vport_stats stats_msg = {}; - struct idpf_vc_xn_params xn_params = {}; - ssize_t reply_sz; + int err; /* Don't send get_stats message if the link is down */ @@ -2816,65 +2428,65 @@ int idpf_send_get_stats_msg(struct idpf_netdev_priv *np, stats_msg.vport_id = cpu_to_le32(np->vport_id); - xn_params.vc_op = VIRTCHNL2_OP_GET_STATS; - xn_params.send_buf.iov_base = &stats_msg; - xn_params.send_buf.iov_len = sizeof(stats_msg); - xn_params.recv_buf = xn_params.send_buf; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; + err = idpf_send_mb_msg_stack(np->adapter, &xn_params, &stats_msg); + if (err) + return err; - reply_sz = idpf_vc_xn_exec(np->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (reply_sz < sizeof(stats_msg)) - return -EIO; + if (xn_params.recv_mem.iov_len < sizeof(*stats_recv)) { + err = -EIO; + goto free_rx_buf; + } + + stats_recv = xn_params.recv_mem.iov_base; spin_lock_bh(&np->stats_lock); - netstats->rx_packets = le64_to_cpu(stats_msg.rx_unicast) + - le64_to_cpu(stats_msg.rx_multicast) + - le64_to_cpu(stats_msg.rx_broadcast); - netstats->tx_packets = le64_to_cpu(stats_msg.tx_unicast) + - le64_to_cpu(stats_msg.tx_multicast) + - le64_to_cpu(stats_msg.tx_broadcast); - netstats->rx_bytes = le64_to_cpu(stats_msg.rx_bytes); - netstats->tx_bytes = le64_to_cpu(stats_msg.tx_bytes); - netstats->rx_errors = le64_to_cpu(stats_msg.rx_errors); - netstats->tx_errors = le64_to_cpu(stats_msg.tx_errors); - netstats->rx_dropped = le64_to_cpu(stats_msg.rx_discards); - netstats->tx_dropped = le64_to_cpu(stats_msg.tx_discards); + netstats->rx_packets = le64_to_cpu(stats_recv->rx_unicast) + + le64_to_cpu(stats_recv->rx_multicast) + + le64_to_cpu(stats_recv->rx_broadcast); + netstats->tx_packets = le64_to_cpu(stats_recv->tx_unicast) + + le64_to_cpu(stats_recv->tx_multicast) + + le64_to_cpu(stats_recv->tx_broadcast); + netstats->rx_bytes = le64_to_cpu(stats_recv->rx_bytes); + netstats->tx_bytes = le64_to_cpu(stats_recv->tx_bytes); + netstats->rx_errors = le64_to_cpu(stats_recv->rx_errors); + netstats->tx_errors = le64_to_cpu(stats_recv->tx_errors); + netstats->rx_dropped = le64_to_cpu(stats_recv->rx_discards); + netstats->tx_dropped = le64_to_cpu(stats_recv->tx_discards); - port_stats->vport_stats = stats_msg; + port_stats->vport_stats = *stats_recv; spin_unlock_bh(&np->stats_lock); - return 0; +free_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + + return err; } /** - * idpf_send_get_set_rss_lut_msg - Send virtchnl get or set RSS lut message + * idpf_send_set_rss_lut_msg - Send virtchnl set RSS lut message * @adapter: adapter pointer used to send virtchnl message * @rss_data: pointer to RSS key and lut info * @vport_id: vport identifier used while preparing the virtchnl message - * @get: flag to set or get RSS look up table * - * When rxhash is disabled, RSS LUT will be configured with zeros. If rxhash + * When rxhash is disabled, RSS LUT will be configured with zeros. If rxhash * is enabled, the LUT values stored in driver's soft copy will be used to setup * the HW. * * Return: 0 on success, negative on failure. */ -int idpf_send_get_set_rss_lut_msg(struct idpf_adapter *adapter, - struct idpf_rss_data *rss_data, - u32 vport_id, bool get) +int idpf_send_set_rss_lut_msg(struct idpf_adapter *adapter, + struct idpf_rss_data *rss_data, u32 vport_id) { - struct virtchnl2_rss_lut *recv_rl __free(kfree) = NULL; - struct virtchnl2_rss_lut *rl __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - int buf_size, lut_buf_size; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_SET_RSS_LUT, + }; + struct virtchnl2_rss_lut *rl; struct idpf_vport *vport; - ssize_t reply_sz; + int buf_size, i, err; bool rxhash_ena; - int i; vport = idpf_vid_to_vport(adapter, vport_id); if (!vport) @@ -2888,76 +2500,36 @@ int idpf_send_get_set_rss_lut_msg(struct idpf_adapter *adapter, return -ENOMEM; rl->vport_id = cpu_to_le32(vport_id); + rl->lut_entries = cpu_to_le16(rss_data->rss_lut_size); + for (i = 0; i < rss_data->rss_lut_size; i++) + rl->lut[i] = rxhash_ena ? cpu_to_le32(rss_data->rss_lut[i]) : 0; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.send_buf.iov_base = rl; - xn_params.send_buf.iov_len = buf_size; + err = idpf_send_mb_msg_kfree(adapter, &xn_params, rl, buf_size); + if (err) + return err; - if (get) { - recv_rl = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL); - if (!recv_rl) - return -ENOMEM; - xn_params.vc_op = VIRTCHNL2_OP_GET_RSS_LUT; - xn_params.recv_buf.iov_base = recv_rl; - xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN; - } else { - rl->lut_entries = cpu_to_le16(rss_data->rss_lut_size); - for (i = 0; i < rss_data->rss_lut_size; i++) - rl->lut[i] = rxhash_ena ? - cpu_to_le32(rss_data->rss_lut[i]) : 0; + libie_ctlq_release_rx_buf(&xn_params.recv_mem); - xn_params.vc_op = VIRTCHNL2_OP_SET_RSS_LUT; - } - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (!get) - return 0; - if (reply_sz < sizeof(struct virtchnl2_rss_lut)) - return -EIO; - - lut_buf_size = le16_to_cpu(recv_rl->lut_entries) * sizeof(u32); - if (reply_sz < lut_buf_size) - return -EIO; - - /* size didn't change, we can reuse existing lut buf */ - if (rss_data->rss_lut_size == le16_to_cpu(recv_rl->lut_entries)) - goto do_memcpy; - - rss_data->rss_lut_size = le16_to_cpu(recv_rl->lut_entries); - kfree(rss_data->rss_lut); - - rss_data->rss_lut = kzalloc(lut_buf_size, GFP_KERNEL); - if (!rss_data->rss_lut) { - rss_data->rss_lut_size = 0; - return -ENOMEM; - } - -do_memcpy: - memcpy(rss_data->rss_lut, recv_rl->lut, rss_data->rss_lut_size); - - return 0; + return err; } /** - * idpf_send_get_set_rss_key_msg - Send virtchnl get or set RSS key message + * idpf_send_set_rss_key_msg - Send virtchnl set RSS key message * @adapter: adapter pointer used to send virtchnl message * @rss_data: pointer to RSS key and lut info * @vport_id: vport identifier used while preparing the virtchnl message - * @get: flag to set or get RSS look up table * * Return: 0 on success, negative on failure */ -int idpf_send_get_set_rss_key_msg(struct idpf_adapter *adapter, - struct idpf_rss_data *rss_data, - u32 vport_id, bool get) +int idpf_send_set_rss_key_msg(struct idpf_adapter *adapter, + struct idpf_rss_data *rss_data, u32 vport_id) { - struct virtchnl2_rss_key *recv_rk __free(kfree) = NULL; - struct virtchnl2_rss_key *rk __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - ssize_t reply_sz; - int i, buf_size; - u16 key_size; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_SET_RSS_KEY, + }; + struct virtchnl2_rss_key *rk; + int i, buf_size, err; buf_size = struct_size(rk, key_flex, rss_data->rss_key_size); rk = kzalloc(buf_size, GFP_KERNEL); @@ -2965,54 +2537,17 @@ int idpf_send_get_set_rss_key_msg(struct idpf_adapter *adapter, return -ENOMEM; rk->vport_id = cpu_to_le32(vport_id); - xn_params.send_buf.iov_base = rk; - xn_params.send_buf.iov_len = buf_size; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - if (get) { - recv_rk = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL); - if (!recv_rk) - return -ENOMEM; + rk->key_len = cpu_to_le16(rss_data->rss_key_size); + for (i = 0; i < rss_data->rss_key_size; i++) + rk->key_flex[i] = rss_data->rss_key[i]; - xn_params.vc_op = VIRTCHNL2_OP_GET_RSS_KEY; - xn_params.recv_buf.iov_base = recv_rk; - xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN; - } else { - rk->key_len = cpu_to_le16(rss_data->rss_key_size); - for (i = 0; i < rss_data->rss_key_size; i++) - rk->key_flex[i] = rss_data->rss_key[i]; + err = idpf_send_mb_msg_kfree(adapter, &xn_params, rk, buf_size); + if (err) + return err; - xn_params.vc_op = VIRTCHNL2_OP_SET_RSS_KEY; - } + libie_ctlq_release_rx_buf(&xn_params.recv_mem); - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (!get) - return 0; - if (reply_sz < sizeof(struct virtchnl2_rss_key)) - return -EIO; - - key_size = min_t(u16, NETDEV_RSS_KEY_LEN, - le16_to_cpu(recv_rk->key_len)); - if (reply_sz < key_size) - return -EIO; - - /* key len didn't change, reuse existing buf */ - if (rss_data->rss_key_size == key_size) - goto do_memcpy; - - rss_data->rss_key_size = key_size; - kfree(rss_data->rss_key); - rss_data->rss_key = kzalloc(key_size, GFP_KERNEL); - if (!rss_data->rss_key) { - rss_data->rss_key_size = 0; - return -ENOMEM; - } - -do_memcpy: - memcpy(rss_data->rss_key, recv_rk->key_flex, rss_data->rss_key_size); - - return 0; + return err; } /** @@ -3189,60 +2724,63 @@ static void idpf_parse_protocol_ids(struct virtchnl2_ptype *ptype, */ static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter) { - struct virtchnl2_get_ptype_info *get_ptype_info __free(kfree) = NULL; - struct virtchnl2_get_ptype_info *ptype_info __free(kfree) = NULL; - struct libeth_rx_pt *singleq_pt_lkup __free(kfree) = NULL; - struct libeth_rx_pt *splitq_pt_lkup __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_GET_PTYPE_INFO, + }; + struct virtchnl2_get_ptype_info *get_ptype_info; + struct virtchnl2_get_ptype_info *ptype_info; + int err = 0, max_ptype = IDPF_RX_MAX_PTYPE; + int buf_size = sizeof(*get_ptype_info); + struct libeth_rx_pt *singleq_pt_lkup; + struct libeth_rx_pt *splitq_pt_lkup; int ptypes_recvd = 0, ptype_offset; - u32 max_ptype = IDPF_RX_MAX_PTYPE; u16 next_ptype_id = 0; - ssize_t reply_sz; singleq_pt_lkup = kzalloc_objs(*singleq_pt_lkup, IDPF_RX_MAX_BASE_PTYPE); if (!singleq_pt_lkup) return -ENOMEM; splitq_pt_lkup = kzalloc_objs(*splitq_pt_lkup, max_ptype); - if (!splitq_pt_lkup) - return -ENOMEM; - - get_ptype_info = kzalloc_obj(*get_ptype_info); - if (!get_ptype_info) - return -ENOMEM; - - ptype_info = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL); - if (!ptype_info) - return -ENOMEM; - - xn_params.vc_op = VIRTCHNL2_OP_GET_PTYPE_INFO; - xn_params.send_buf.iov_base = get_ptype_info; - xn_params.send_buf.iov_len = sizeof(*get_ptype_info); - xn_params.recv_buf.iov_base = ptype_info; - xn_params.recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; + if (!splitq_pt_lkup) { + err = -ENOMEM; + goto free_singleq; + } while (next_ptype_id < max_ptype) { + u16 num_ptypes; + + get_ptype_info = kzalloc(buf_size, GFP_KERNEL); + if (!get_ptype_info) { + err = -ENOMEM; + goto free_splitq; + } + get_ptype_info->start_ptype_id = cpu_to_le16(next_ptype_id); if ((next_ptype_id + IDPF_RX_MAX_PTYPES_PER_BUF) > max_ptype) - get_ptype_info->num_ptypes = - cpu_to_le16(max_ptype - next_ptype_id); + num_ptypes = max_ptype - next_ptype_id; else - get_ptype_info->num_ptypes = - cpu_to_le16(IDPF_RX_MAX_PTYPES_PER_BUF); + num_ptypes = IDPF_RX_MAX_PTYPES_PER_BUF; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + get_ptype_info->num_ptypes = cpu_to_le16(num_ptypes); + err = idpf_send_mb_msg_kfree(adapter, &xn_params, + get_ptype_info, buf_size); + if (err) + goto free_splitq; + ptype_info = xn_params.recv_mem.iov_base; + if (xn_params.recv_mem.iov_len < sizeof(*ptype_info)) { + err = -EIO; + goto free_rx_buf; + } ptypes_recvd += le16_to_cpu(ptype_info->num_ptypes); - if (ptypes_recvd > max_ptype) - return -EINVAL; - - next_ptype_id = le16_to_cpu(get_ptype_info->start_ptype_id) + - le16_to_cpu(get_ptype_info->num_ptypes); + if (ptypes_recvd > max_ptype) { + err = -EINVAL; + goto free_rx_buf; + } + next_ptype_id = next_ptype_id + num_ptypes; ptype_offset = IDPF_RX_PTYPE_HDR_SZ; for (u16 i = 0; i < le16_to_cpu(ptype_info->num_ptypes); i++) { @@ -3252,19 +2790,28 @@ static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter) ptype = (struct virtchnl2_ptype *) ((u8 *)ptype_info + ptype_offset); + if (xn_params.recv_mem.iov_len < + ptype_offset + sizeof(struct virtchnl2_ptype)) { + err = -EINVAL; + goto free_rx_buf; + } pt_10 = le16_to_cpu(ptype->ptype_id_10); pt_8 = ptype->ptype_id_8; ptype_offset += IDPF_GET_PTYPE_SIZE(ptype); - if (ptype_offset > IDPF_CTLQ_MAX_BUF_LEN) - return -EINVAL; + if (xn_params.recv_mem.iov_len < ptype_offset) { + err = -EINVAL; + goto free_rx_buf; + } /* 0xFFFF indicates end of ptypes */ if (pt_10 == IDPF_INVALID_PTYPE_ID) goto out; - if (pt_10 >= max_ptype) - return -EINVAL; + if (pt_10 >= max_ptype) { + err = -EINVAL; + goto free_rx_buf; + } idpf_parse_protocol_ids(ptype, &rx_pt); idpf_finalize_ptype_lookup(&rx_pt); @@ -3278,13 +2825,24 @@ static int idpf_send_get_rx_ptype_msg(struct idpf_adapter *adapter) if (!singleq_pt_lkup[pt_8].outer_ip) singleq_pt_lkup[pt_8] = rx_pt; } + + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + xn_params.recv_mem = (struct kvec) {}; } out: - adapter->splitq_pt_lkup = no_free_ptr(splitq_pt_lkup); - adapter->singleq_pt_lkup = no_free_ptr(singleq_pt_lkup); + adapter->splitq_pt_lkup = splitq_pt_lkup; + adapter->singleq_pt_lkup = singleq_pt_lkup; + splitq_pt_lkup = NULL; + singleq_pt_lkup = NULL; +free_rx_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); +free_splitq: + kfree(splitq_pt_lkup); +free_singleq: + kfree(singleq_pt_lkup); - return 0; + return err; } /** @@ -3312,40 +2870,23 @@ static void idpf_rel_rx_pt_lkup(struct idpf_adapter *adapter) int idpf_send_ena_dis_loopback_msg(struct idpf_adapter *adapter, u32 vport_id, bool loopback_ena) { - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_LOOPBACK, + }; struct virtchnl2_loopback loopback; - ssize_t reply_sz; + int err; loopback.vport_id = cpu_to_le32(vport_id); loopback.enable = loopback_ena; - xn_params.vc_op = VIRTCHNL2_OP_LOOPBACK; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.send_buf.iov_base = &loopback; - xn_params.send_buf.iov_len = sizeof(loopback); - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); + err = idpf_send_mb_msg_stack(adapter, &xn_params, &loopback); + if (err) + return err; - return reply_sz < 0 ? reply_sz : 0; -} + libie_ctlq_release_rx_buf(&xn_params.recv_mem); -/** - * idpf_find_ctlq - Given a type and id, find ctlq info - * @hw: hardware struct - * @type: type of ctrlq to find - * @id: ctlq id to find - * - * Returns pointer to found ctlq info struct, NULL otherwise. - */ -static struct idpf_ctlq_info *idpf_find_ctlq(struct idpf_hw *hw, - enum idpf_ctlq_type type, int id) -{ - struct idpf_ctlq_info *cq, *tmp; - - list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list) - if (cq->q_id == id && cq->cq_type == type) - return cq; - - return NULL; + return 0; } /** @@ -3356,42 +2897,49 @@ static struct idpf_ctlq_info *idpf_find_ctlq(struct idpf_hw *hw, */ int idpf_init_dflt_mbx(struct idpf_adapter *adapter) { - struct idpf_ctlq_create_info ctlq_info[] = { + struct libie_ctlq_ctx *ctx = &adapter->ctlq_ctx; + struct libie_ctlq_create_info ctlq_info[] = { { - .type = IDPF_CTLQ_TYPE_MAILBOX_TX, - .id = IDPF_DFLT_MBX_ID, + .type = LIBIE_CTLQ_TYPE_TX, + .id = LIBIE_CTLQ_MBX_ID, .len = IDPF_DFLT_MBX_Q_LEN, - .buf_size = IDPF_CTLQ_MAX_BUF_LEN }, { - .type = IDPF_CTLQ_TYPE_MAILBOX_RX, - .id = IDPF_DFLT_MBX_ID, + .type = LIBIE_CTLQ_TYPE_RX, + .id = LIBIE_CTLQ_MBX_ID, .len = IDPF_DFLT_MBX_Q_LEN, - .buf_size = IDPF_CTLQ_MAX_BUF_LEN } }; - struct idpf_hw *hw = &adapter->hw; + struct libie_ctlq_xn_init_params params = { + .num_qs = IDPF_NUM_DFLT_MBX_Q, + .cctlq_info = ctlq_info, + .ctx = ctx, + }; int err; - adapter->dev_ops.reg_ops.ctlq_reg_init(adapter, ctlq_info); + adapter->dev_ops.reg_ops.ctlq_reg_init(&ctx->mmio_info, + params.cctlq_info); - err = idpf_ctlq_init(hw, IDPF_NUM_DFLT_MBX_Q, ctlq_info); + err = libie_ctlq_xn_init(¶ms); if (err) return err; - hw->asq = idpf_find_ctlq(hw, IDPF_CTLQ_TYPE_MAILBOX_TX, - IDPF_DFLT_MBX_ID); - hw->arq = idpf_find_ctlq(hw, IDPF_CTLQ_TYPE_MAILBOX_RX, - IDPF_DFLT_MBX_ID); - - if (!hw->asq || !hw->arq) { - idpf_ctlq_deinit(hw); - + adapter->asq = libie_find_ctlq(ctx, LIBIE_CTLQ_TYPE_TX, + LIBIE_CTLQ_MBX_ID); + adapter->arq = libie_find_ctlq(ctx, LIBIE_CTLQ_TYPE_RX, + LIBIE_CTLQ_MBX_ID); + if (!adapter->asq || !adapter->arq) { + adapter->asq = NULL; + adapter->arq = NULL; + libie_ctlq_xn_deinit(params.xnm, ctx); return -ENOENT; } + adapter->xnm = params.xnm; adapter->state = __IDPF_VER_CHECK; + queue_delayed_work(adapter->mbx_wq, &adapter->mbx_task, 0); + return 0; } @@ -3401,12 +2949,18 @@ int idpf_init_dflt_mbx(struct idpf_adapter *adapter) */ void idpf_deinit_dflt_mbx(struct idpf_adapter *adapter) { - if (adapter->hw.arq && adapter->hw.asq) { - idpf_mb_clean(adapter, adapter->hw.asq); - idpf_ctlq_deinit(&adapter->hw); + idpf_mb_intr_rel_irq(adapter); + cancel_delayed_work_sync(&adapter->mbx_task); + + if (adapter->xnm) { + libie_ctlq_xn_shutdown(adapter->xnm); + idpf_mb_clean(adapter->asq, true); + libie_ctlq_xn_deinit(adapter->xnm, &adapter->ctlq_ctx); } - adapter->hw.arq = NULL; - adapter->hw.asq = NULL; + + adapter->arq = NULL; + adapter->asq = NULL; + adapter->xnm = NULL; } /** @@ -3419,8 +2973,6 @@ static void idpf_vport_params_buf_rel(struct idpf_adapter *adapter) { kfree(adapter->vport_params_recvd); adapter->vport_params_recvd = NULL; - kfree(adapter->vport_params_reqd); - adapter->vport_params_reqd = NULL; kfree(adapter->vport_ids); adapter->vport_ids = NULL; } @@ -3435,15 +2987,10 @@ static int idpf_vport_params_buf_alloc(struct idpf_adapter *adapter) { u16 num_max_vports = idpf_get_max_vports(adapter); - adapter->vport_params_reqd = kzalloc_objs(*adapter->vport_params_reqd, - num_max_vports); - if (!adapter->vport_params_reqd) - return -ENOMEM; - adapter->vport_params_recvd = kzalloc_objs(*adapter->vport_params_recvd, num_max_vports); if (!adapter->vport_params_recvd) - goto err_mem; + return -ENOMEM; adapter->vport_ids = kcalloc(num_max_vports, sizeof(u32), GFP_KERNEL); if (!adapter->vport_ids) @@ -3484,15 +3031,6 @@ int idpf_vc_core_init(struct idpf_adapter *adapter) u16 num_max_vports; int err = 0; - if (!adapter->vcxn_mngr) { - adapter->vcxn_mngr = kzalloc_obj(*adapter->vcxn_mngr); - if (!adapter->vcxn_mngr) { - err = -ENOMEM; - goto init_failed; - } - } - idpf_vc_xn_init(adapter->vcxn_mngr); - while (adapter->state != __IDPF_INIT_SW) { switch (adapter->state) { case __IDPF_VER_CHECK: @@ -3531,34 +3069,29 @@ int idpf_vc_core_init(struct idpf_adapter *adapter) } if (idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_LAN_MEMORY_REGIONS)) { - err = idpf_send_get_lan_memory_regions(adapter); + err = idpf_cfg_lan_memory_regions(adapter); if (err) { - dev_err(&adapter->pdev->dev, "Failed to get LAN memory regions: %d\n", + dev_err(&adapter->pdev->dev, "Failed to configure LAN memory regions: %d\n", err); return -EINVAL; } } else { /* Fallback to mapping the remaining regions of the entire BAR */ - err = idpf_calc_remaining_mmio_regs(adapter); + err = idpf_map_remaining_mmio_regs(adapter); if (err) { - dev_err(&adapter->pdev->dev, "Failed to allocate BAR0 region(s): %d\n", + dev_err(&adapter->pdev->dev, "Failed to configure BAR0 region(s): %d\n", err); - return -ENOMEM; + return err; } } - err = idpf_map_lan_mmio_regs(adapter); - if (err) { - dev_err(&adapter->pdev->dev, "Failed to map BAR0 region(s): %d\n", - err); - return -ENOMEM; - } - pci_sriov_set_totalvfs(adapter->pdev, idpf_get_max_vfs(adapter)); num_max_vports = idpf_get_max_vports(adapter); adapter->vports = kzalloc_objs(*adapter->vports, num_max_vports); - if (!adapter->vports) - return -ENOMEM; + if (!adapter->vports) { + err = -ENOMEM; + goto decfg_regions; + } if (!adapter->netdevs) { adapter->netdevs = kzalloc_objs(struct net_device *, @@ -3630,6 +3163,8 @@ int idpf_vc_core_init(struct idpf_adapter *adapter) err_netdev_alloc: kfree(adapter->vports); adapter->vports = NULL; +decfg_regions: + idpf_decfg_lan_memory_regions(adapter); return err; init_failed: @@ -3647,8 +3182,7 @@ int idpf_vc_core_init(struct idpf_adapter *adapter) * the mailbox again */ adapter->state = __IDPF_VER_CHECK; - if (adapter->vcxn_mngr) - idpf_vc_xn_shutdown(adapter->vcxn_mngr); + libie_ctlq_xn_shutdown(adapter->xnm); set_bit(IDPF_HR_DRV_LOAD, adapter->flags); queue_delayed_work(adapter->vc_event_wq, &adapter->vc_event_task, msecs_to_jiffies(task_delay)); @@ -3663,7 +3197,6 @@ int idpf_vc_core_init(struct idpf_adapter *adapter) */ void idpf_vc_core_deinit(struct idpf_adapter *adapter) { - struct idpf_hw *hw = &adapter->hw; bool remove_in_prog; if (!test_bit(IDPF_VC_CORE_INIT, adapter->flags)) @@ -3672,7 +3205,7 @@ void idpf_vc_core_deinit(struct idpf_adapter *adapter) /* Avoid transaction timeouts when called during reset */ remove_in_prog = test_bit(IDPF_REMOVE_IN_PROG, adapter->flags); if (!remove_in_prog) - idpf_vc_xn_shutdown(adapter->vcxn_mngr); + libie_ctlq_xn_shutdown(adapter->xnm); idpf_ptp_release(adapter); idpf_deinit_task(adapter); @@ -3681,19 +3214,17 @@ void idpf_vc_core_deinit(struct idpf_adapter *adapter) idpf_intr_rel(adapter); if (remove_in_prog) - idpf_vc_xn_shutdown(adapter->vcxn_mngr); + libie_ctlq_xn_shutdown(adapter->xnm); cancel_delayed_work_sync(&adapter->serv_task); cancel_delayed_work_sync(&adapter->mbx_task); idpf_vport_params_buf_rel(adapter); - kfree(hw->lan_regs); - hw->lan_regs = NULL; - kfree(adapter->vports); adapter->vports = NULL; + idpf_decfg_lan_memory_regions(adapter); clear_bit(IDPF_VC_CORE_INIT, adapter->flags); } @@ -4220,9 +3751,9 @@ static void idpf_set_mac_type(const u8 *default_mac_addr, /** * idpf_mac_filter_async_handler - Async callback for mac filters - * @adapter: private data struct - * @xn: transaction for message - * @ctlq_msg: received message + * @ctx: controlq context structure + * @buff: response buffer pointer and size + * @status: async call return value * * In some scenarios driver can't sleep and wait for a reply (e.g.: stack is * holding rtnl_lock) when adding a new mac filter. It puts us in a difficult @@ -4230,13 +3761,14 @@ static void idpf_set_mac_type(const u8 *default_mac_addr, * ultimately do is remove it from our list of mac filters and report the * error. */ -static int idpf_mac_filter_async_handler(struct idpf_adapter *adapter, - struct idpf_vc_xn *xn, - const struct idpf_ctlq_msg *ctlq_msg) +static void idpf_mac_filter_async_handler(void *ctx, + struct kvec *buff, + int status) { struct virtchnl2_mac_addr_list *ma_list; struct idpf_vport_config *vport_config; struct virtchnl2_mac_addr *mac_addr; + struct idpf_adapter *adapter = ctx; struct idpf_mac_filter *f, *tmp; struct list_head *ma_list_head; struct idpf_vport *vport; @@ -4244,18 +3776,18 @@ static int idpf_mac_filter_async_handler(struct idpf_adapter *adapter, int i; /* if success we're done, we're only here if something bad happened */ - if (!ctlq_msg->cookie.mbx.chnl_retval) - return 0; + if (!status || status == -ETIMEDOUT) + return; + ma_list = buff->iov_base; /* make sure at least struct is there */ - if (xn->reply_sz < sizeof(*ma_list)) + if (buff->iov_len < sizeof(*ma_list)) goto invalid_payload; - ma_list = ctlq_msg->ctx.indirect.payload->va; mac_addr = ma_list->mac_addr_list; num_entries = le16_to_cpu(ma_list->num_mac_addr); /* we should have received a buffer at least this big */ - if (xn->reply_sz < struct_size(ma_list, mac_addr_list, num_entries)) + if (buff->iov_len < struct_size(ma_list, mac_addr_list, num_entries)) goto invalid_payload; vport = idpf_vid_to_vport(adapter, le32_to_cpu(ma_list->vport_id)); @@ -4275,16 +3807,13 @@ static int idpf_mac_filter_async_handler(struct idpf_adapter *adapter, if (ether_addr_equal(mac_addr[i].addr, f->macaddr)) list_del(&f->list); spin_unlock_bh(&vport_config->mac_filter_list_lock); - dev_err_ratelimited(&adapter->pdev->dev, "Received error sending MAC filter request (op %d)\n", - xn->vc_op); - - return 0; + dev_err_ratelimited(&adapter->pdev->dev, "Received error %d on sending MAC filter request\n", + status); + return; invalid_payload: - dev_err_ratelimited(&adapter->pdev->dev, "Received invalid MAC filter payload (op %d) (len %zd)\n", - xn->vc_op, xn->reply_sz); - - return -EINVAL; + dev_err_ratelimited(&adapter->pdev->dev, "Received invalid MAC filter payload (len %zd)\n", + buff->iov_len); } /** @@ -4303,19 +3832,21 @@ int idpf_add_del_mac_filters(struct idpf_adapter *adapter, const u8 *default_mac_addr, u32 vport_id, bool add, bool async) { - struct virtchnl2_mac_addr_list *ma_list __free(kfree) = NULL; struct virtchnl2_mac_addr *mac_addr __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = add ? VIRTCHNL2_OP_ADD_MAC_ADDR : + VIRTCHNL2_OP_DEL_MAC_ADDR, + }; + struct virtchnl2_mac_addr_list *ma_list; u32 num_msgs, total_filters = 0; struct idpf_mac_filter *f; - ssize_t reply_sz; - int i = 0, k; + int i = 0; - xn_params.vc_op = add ? VIRTCHNL2_OP_ADD_MAC_ADDR : - VIRTCHNL2_OP_DEL_MAC_ADDR; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.async = async; - xn_params.async_handler = idpf_mac_filter_async_handler; + if (async) { + xn_params.resp_cb = idpf_mac_filter_async_handler; + xn_params.send_ctx = adapter; + } spin_lock_bh(&vport_config->mac_filter_list_lock); @@ -4370,32 +3901,31 @@ int idpf_add_del_mac_filters(struct idpf_adapter *adapter, */ num_msgs = DIV_ROUND_UP(total_filters, IDPF_NUM_FILTERS_PER_MSG); - for (i = 0, k = 0; i < num_msgs; i++) { - u32 entries_size, buf_size, num_entries; + for (u32 i = 0, k = 0; i < num_msgs; i++) { + u32 entries_size, num_entries; + size_t buf_size; + int err; num_entries = min_t(u32, total_filters, IDPF_NUM_FILTERS_PER_MSG); entries_size = sizeof(struct virtchnl2_mac_addr) * num_entries; buf_size = struct_size(ma_list, mac_addr_list, num_entries); - if (!ma_list || num_entries != IDPF_NUM_FILTERS_PER_MSG) { - kfree(ma_list); - ma_list = kzalloc(buf_size, GFP_ATOMIC); - if (!ma_list) - return -ENOMEM; - } else { - memset(ma_list, 0, buf_size); - } + ma_list = kzalloc(buf_size, GFP_ATOMIC); + if (!ma_list) + return -ENOMEM; ma_list->vport_id = cpu_to_le32(vport_id); ma_list->num_mac_addr = cpu_to_le16(num_entries); memcpy(ma_list->mac_addr_list, &mac_addr[k], entries_size); - xn_params.send_buf.iov_base = ma_list; - xn_params.send_buf.iov_len = buf_size; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + err = idpf_send_mb_msg_kfree(adapter, &xn_params, ma_list, + buf_size); + if (err) + return err; + + if (!async) + libie_ctlq_release_rx_buf(&xn_params.recv_mem); k += num_entries; total_filters -= num_entries; @@ -4404,6 +3934,26 @@ int idpf_add_del_mac_filters(struct idpf_adapter *adapter, return 0; } +/** + * idpf_promiscuous_async_handler - async callback for promiscuous mode + * @ctx: controlq context structure + * @buff: response buffer pointer and size + * @status: async call return value + * + * Nobody is waiting for the promiscuous virtchnl message response. Print + * an error message if something went wrong and return. + */ +static void idpf_promiscuous_async_handler(void *ctx, + struct kvec *buff, + int status) +{ + struct idpf_adapter *adapter = ctx; + + if (status) + dev_err_ratelimited(&adapter->pdev->dev, "Failed to set promiscuous mode: %d\n", + status); +} + /** * idpf_set_promiscuous - set promiscuous and send message to mailbox * @adapter: Driver specific private structure @@ -4418,9 +3968,13 @@ int idpf_set_promiscuous(struct idpf_adapter *adapter, struct idpf_vport_user_config_data *config_data, u32 vport_id) { - struct idpf_vc_xn_params xn_params = {}; + struct libie_ctlq_xn_send_params xn_params = { + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + .chnl_opcode = VIRTCHNL2_OP_CONFIG_PROMISCUOUS_MODE, + .resp_cb = idpf_promiscuous_async_handler, + .send_ctx = adapter, + }; struct virtchnl2_promisc_info vpi; - ssize_t reply_sz; u16 flags = 0; if (test_bit(__IDPF_PROMISC_UC, config_data->user_flags)) @@ -4431,15 +3985,7 @@ int idpf_set_promiscuous(struct idpf_adapter *adapter, vpi.vport_id = cpu_to_le32(vport_id); vpi.flags = cpu_to_le16(flags); - xn_params.vc_op = VIRTCHNL2_OP_CONFIG_PROMISCUOUS_MODE; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.send_buf.iov_base = &vpi; - xn_params.send_buf.iov_len = sizeof(vpi); - /* setting promiscuous is only ever done asynchronously */ - xn_params.async = true; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - - return reply_sz < 0 ? reply_sz : 0; + return idpf_send_mb_msg_stack(adapter, &xn_params, &vpi); } /** @@ -4448,7 +3994,7 @@ int idpf_set_promiscuous(struct idpf_adapter *adapter, * @send_msg: message to send * @msg_size: size of message to send * @recv_msg: message to populate on reception of response - * @recv_len: length of message copied into recv_msg or 0 on error + * @recv_len: on input, maximum response size; on success, actual response size * * Return: 0 on success or error code on failure. */ @@ -4457,26 +4003,39 @@ int idpf_idc_rdma_vc_send_sync(struct iidc_rdma_core_dev_info *cdev_info, u8 *recv_msg, u16 *recv_len) { struct idpf_adapter *adapter = pci_get_drvdata(cdev_info->pdev); - struct idpf_vc_xn_params xn_params = { }; - ssize_t reply_sz; - u16 recv_size; + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_RDMA, + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + }; + u8 on_stack_buf[LIBIE_CP_TX_COPYBREAK]; + void *send_buf; + int err; - if (!recv_msg || !recv_len || msg_size > IDPF_CTLQ_MAX_BUF_LEN) + if (!recv_msg || !recv_len || msg_size > LIBIE_CTLQ_MAX_BUF_LEN) return -EINVAL; - recv_size = min_t(u16, *recv_len, IDPF_CTLQ_MAX_BUF_LEN); - *recv_len = 0; - xn_params.vc_op = VIRTCHNL2_OP_RDMA; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - xn_params.send_buf.iov_base = send_msg; - xn_params.send_buf.iov_len = msg_size; - xn_params.recv_buf.iov_base = recv_msg; - xn_params.recv_buf.iov_len = recv_size; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - *recv_len = reply_sz; + if (!libie_cp_can_send_onstack(msg_size)) { + send_buf = kzalloc(msg_size, GFP_KERNEL); + if (!send_buf) + return -ENOMEM; + } else { + send_buf = on_stack_buf; + } - return 0; + memcpy(send_buf, send_msg, msg_size); + err = idpf_send_mb_msg(adapter, &xn_params, send_buf, msg_size); + if (err) + return err; + + if (xn_params.recv_mem.iov_len > *recv_len) { + err = -EINVAL; + goto rel_buf; + } + + *recv_len = xn_params.recv_mem.iov_len; + memcpy(recv_msg, xn_params.recv_mem.iov_base, *recv_len); +rel_buf: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + return err; } EXPORT_SYMBOL_GPL(idpf_idc_rdma_vc_send_sync); diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h index 9b1c9c86f6ea..5d27805ff40f 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h @@ -4,89 +4,9 @@ #ifndef _IDPF_VIRTCHNL_H_ #define _IDPF_VIRTCHNL_H_ -#include "virtchnl2.h" +#include #define IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC (60 * 1000) -#define IDPF_VC_XN_IDX_M GENMASK(7, 0) -#define IDPF_VC_XN_SALT_M GENMASK(15, 8) -#define IDPF_VC_XN_RING_LEN U8_MAX - -/** - * enum idpf_vc_xn_state - Virtchnl transaction status - * @IDPF_VC_XN_IDLE: not expecting a reply, ready to be used - * @IDPF_VC_XN_WAITING: expecting a reply, not yet received - * @IDPF_VC_XN_COMPLETED_SUCCESS: a reply was expected and received, buffer - * updated - * @IDPF_VC_XN_COMPLETED_FAILED: a reply was expected and received, but there - * was an error, buffer not updated - * @IDPF_VC_XN_SHUTDOWN: transaction object cannot be used, VC torn down - * @IDPF_VC_XN_ASYNC: transaction sent asynchronously and doesn't have the - * return context; a callback may be provided to handle - * return - */ -enum idpf_vc_xn_state { - IDPF_VC_XN_IDLE = 1, - IDPF_VC_XN_WAITING, - IDPF_VC_XN_COMPLETED_SUCCESS, - IDPF_VC_XN_COMPLETED_FAILED, - IDPF_VC_XN_SHUTDOWN, - IDPF_VC_XN_ASYNC, -}; - -struct idpf_vc_xn; -/* Callback for asynchronous messages */ -typedef int (*async_vc_cb) (struct idpf_adapter *, struct idpf_vc_xn *, - const struct idpf_ctlq_msg *); - -/** - * struct idpf_vc_xn - Data structure representing virtchnl transactions - * @completed: virtchnl event loop uses that to signal when a reply is - * available, uses kernel completion API - * @lock: protects the transaction state fields below - * @state: virtchnl event loop stores the data below, protected by @lock - * @reply_sz: Original size of reply, may be > reply_buf.iov_len; it will be - * truncated on its way to the receiver thread according to - * reply_buf.iov_len. - * @reply: Reference to the buffer(s) where the reply data should be written - * to. May be 0-length (then NULL address permitted) if the reply data - * should be ignored. - * @async_handler: if sent asynchronously, a callback can be provided to handle - * the reply when it's received - * @vc_op: corresponding opcode sent with this transaction - * @idx: index used as retrieval on reply receive, used for cookie - * @salt: changed every message to make unique, used for cookie - */ -struct idpf_vc_xn { - struct completion completed; - spinlock_t lock; - enum idpf_vc_xn_state state; - size_t reply_sz; - struct kvec reply; - async_vc_cb async_handler; - u32 vc_op; - u8 idx; - u8 salt; -}; - -/** - * struct idpf_vc_xn_params - Parameters for executing transaction - * @send_buf: kvec for send buffer - * @recv_buf: kvec for recv buffer, may be NULL, must then have zero length - * @timeout_ms: timeout to wait for reply - * @async: send message asynchronously, will not wait on completion - * @async_handler: If sent asynchronously, optional callback handler. The user - * must be careful when using async handlers as the memory for - * the recv_buf _cannot_ be on stack if this is async. - * @vc_op: virtchnl op to send - */ -struct idpf_vc_xn_params { - struct kvec send_buf; - struct kvec recv_buf; - int timeout_ms; - bool async; - async_vc_cb async_handler; - u32 vc_op; -}; struct idpf_adapter; struct idpf_netdev_priv; @@ -96,8 +16,6 @@ struct idpf_vport_max_q; struct idpf_vport_config; struct idpf_vport_user_config_data; -ssize_t idpf_vc_xn_exec(struct idpf_adapter *adapter, - const struct idpf_vc_xn_params *params); int idpf_init_dflt_mbx(struct idpf_adapter *adapter); void idpf_deinit_dflt_mbx(struct idpf_adapter *adapter); int idpf_vc_core_init(struct idpf_adapter *adapter); @@ -124,9 +42,35 @@ bool idpf_sideband_action_ena(struct idpf_vport *vport, struct ethtool_rx_flow_spec *fsp); unsigned int idpf_fsteer_max_rules(struct idpf_vport *vport); -int idpf_recv_mb_msg(struct idpf_adapter *adapter, struct idpf_ctlq_info *arq); -int idpf_send_mb_msg(struct idpf_adapter *adapter, struct idpf_ctlq_info *asq, - u32 op, u16 msg_size, u8 *msg, u16 cookie); +void idpf_recv_event_msg(struct libie_ctlq_ctx *ctx, + struct libie_ctlq_msg *ctlq_msg); +int idpf_send_mb_msg(struct idpf_adapter *adapter, + struct libie_ctlq_xn_send_params *xn_params, + void *send_buf, size_t send_buf_size); +int idpf_send_mb_msg_kfree(struct idpf_adapter *adapter, + struct libie_ctlq_xn_send_params *xn_params, + void *send_buf, size_t send_buf_size); +void idpf_send_vf_reset_msg(struct idpf_adapter *adapter); +bool idpf_mmio_region_non_static(struct libie_mmio_info *mmio_info, + struct libie_pci_mmio_region *reg); + +/** + * idpf_send_mb_msg_stack - send a mailbox message from an on-stack buffer + * @adapter: driver specific private structure + * @xn_params: Xn send parameters to fill + * @ptr: pointer to the on-stack message object to send + * + * Send size is deduced based on the pointer type. + * + * Return: %0 on success, -%errno on failure. + */ +#define idpf_send_mb_msg_stack(adapter, xn_params, ptr) \ +({ \ + typeof(ptr) __ptr = (ptr); \ + \ + static_assert(sizeof(*__ptr) <= LIBIE_CP_TX_COPYBREAK); \ + idpf_send_mb_msg(adapter, xn_params, __ptr, sizeof(*__ptr)); \ +}) struct idpf_queue_ptr { enum virtchnl2_queue_type type; @@ -208,13 +152,10 @@ int idpf_send_ena_dis_loopback_msg(struct idpf_adapter *adapter, u32 vport_id, int idpf_send_get_stats_msg(struct idpf_netdev_priv *np, struct idpf_port_stats *port_stats); int idpf_send_set_sriov_vfs_msg(struct idpf_adapter *adapter, u16 num_vfs); -int idpf_send_get_set_rss_key_msg(struct idpf_adapter *adapter, - struct idpf_rss_data *rss_data, - u32 vport_id, bool get); -int idpf_send_get_set_rss_lut_msg(struct idpf_adapter *adapter, - struct idpf_rss_data *rss_data, - u32 vport_id, bool get); -void idpf_vc_xn_shutdown(struct idpf_vc_xn_manager *vcxn_mngr); +int idpf_send_set_rss_key_msg(struct idpf_adapter *adapter, + struct idpf_rss_data *rss_data, u32 vport_id); +int idpf_send_set_rss_lut_msg(struct idpf_adapter *adapter, + struct idpf_rss_data *rss_data, u32 vport_id); int idpf_idc_rdma_vc_send_sync(struct iidc_rdma_core_dev_info *cdev_info, u8 *send_msg, u16 msg_size, u8 *recv_msg, u16 *recv_len); diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c index d9bcc3f61c65..14dba40a993f 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c @@ -15,7 +15,6 @@ */ int idpf_ptp_get_caps(struct idpf_adapter *adapter) { - struct virtchnl2_ptp_get_caps *recv_ptp_caps_msg __free(kfree) = NULL; struct virtchnl2_ptp_get_caps send_ptp_caps_msg = { .caps = cpu_to_le32(VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME | VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME_MB | @@ -24,33 +23,33 @@ int idpf_ptp_get_caps(struct idpf_adapter *adapter) VIRTCHNL2_CAP_PTP_ADJ_DEVICE_CLK_MB | VIRTCHNL2_CAP_PTP_TX_TSTAMPS_MB) }; - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_PTP_GET_CAPS, - .send_buf.iov_base = &send_ptp_caps_msg, - .send_buf.iov_len = sizeof(send_ptp_caps_msg), + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_PTP_GET_CAPS, .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, }; struct virtchnl2_ptp_cross_time_reg_offsets cross_tstamp_offsets; + struct libie_mmio_info *mmio = &adapter->ctlq_ctx.mmio_info; struct virtchnl2_ptp_clk_adj_reg_offsets clk_adj_offsets; struct virtchnl2_ptp_clk_reg_offsets clock_offsets; + struct virtchnl2_ptp_get_caps *recv_ptp_caps_msg; struct idpf_ptp_secondary_mbx *scnd_mbx; struct idpf_ptp *ptp = adapter->ptp; enum idpf_ptp_access access_type; u32 temp_offset; - int reply_sz; + size_t reply_sz; + int err; - recv_ptp_caps_msg = kzalloc_obj(struct virtchnl2_ptp_get_caps); - if (!recv_ptp_caps_msg) - return -ENOMEM; + err = idpf_send_mb_msg_stack(adapter, &xn_params, &send_ptp_caps_msg); + if (err) + return err; - xn_params.recv_buf.iov_base = recv_ptp_caps_msg; - xn_params.recv_buf.iov_len = sizeof(*recv_ptp_caps_msg); + reply_sz = xn_params.recv_mem.iov_len; + if (reply_sz != sizeof(*recv_ptp_caps_msg)) { + err = -EIO; + goto free_resp; + } - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - else if (reply_sz != sizeof(*recv_ptp_caps_msg)) - return -EIO; + recv_ptp_caps_msg = xn_params.recv_mem.iov_base; ptp->caps = le32_to_cpu(recv_ptp_caps_msg->caps); ptp->base_incval = le64_to_cpu(recv_ptp_caps_msg->base_incval); @@ -76,19 +75,20 @@ int idpf_ptp_get_caps(struct idpf_adapter *adapter) clock_offsets = recv_ptp_caps_msg->clk_offsets; temp_offset = le32_to_cpu(clock_offsets.dev_clk_ns_l); - ptp->dev_clk_regs.dev_clk_ns_l = idpf_get_reg_addr(adapter, - temp_offset); + ptp->dev_clk_regs.dev_clk_ns_l = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clock_offsets.dev_clk_ns_h); - ptp->dev_clk_regs.dev_clk_ns_h = idpf_get_reg_addr(adapter, - temp_offset); + ptp->dev_clk_regs.dev_clk_ns_h = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clock_offsets.phy_clk_ns_l); - ptp->dev_clk_regs.phy_clk_ns_l = idpf_get_reg_addr(adapter, - temp_offset); + ptp->dev_clk_regs.phy_clk_ns_l = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clock_offsets.phy_clk_ns_h); - ptp->dev_clk_regs.phy_clk_ns_h = idpf_get_reg_addr(adapter, - temp_offset); + ptp->dev_clk_regs.phy_clk_ns_h = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clock_offsets.cmd_sync_trigger); - ptp->dev_clk_regs.cmd_sync = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.cmd_sync = + libie_pci_get_mmio_addr(mmio, temp_offset); cross_tstamp: access_type = ptp->get_cross_tstamp_access; @@ -98,48 +98,54 @@ int idpf_ptp_get_caps(struct idpf_adapter *adapter) cross_tstamp_offsets = recv_ptp_caps_msg->cross_time_offsets; temp_offset = le32_to_cpu(cross_tstamp_offsets.sys_time_ns_l); - ptp->dev_clk_regs.sys_time_ns_l = idpf_get_reg_addr(adapter, - temp_offset); + ptp->dev_clk_regs.sys_time_ns_l = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(cross_tstamp_offsets.sys_time_ns_h); - ptp->dev_clk_regs.sys_time_ns_h = idpf_get_reg_addr(adapter, - temp_offset); + ptp->dev_clk_regs.sys_time_ns_h = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(cross_tstamp_offsets.cmd_sync_trigger); - ptp->dev_clk_regs.cmd_sync = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.cmd_sync = + libie_pci_get_mmio_addr(mmio, temp_offset); discipline_clock: access_type = ptp->adj_dev_clk_time_access; if (access_type != IDPF_PTP_DIRECT) - return 0; + goto free_resp; clk_adj_offsets = recv_ptp_caps_msg->clk_adj_offsets; /* Device clock offsets */ temp_offset = le32_to_cpu(clk_adj_offsets.dev_clk_cmd_type); - ptp->dev_clk_regs.cmd = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.cmd = libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clk_adj_offsets.dev_clk_incval_l); - ptp->dev_clk_regs.incval_l = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.incval_l = libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clk_adj_offsets.dev_clk_incval_h); - ptp->dev_clk_regs.incval_h = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.incval_h = libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clk_adj_offsets.dev_clk_shadj_l); - ptp->dev_clk_regs.shadj_l = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.shadj_l = libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clk_adj_offsets.dev_clk_shadj_h); - ptp->dev_clk_regs.shadj_h = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.shadj_h = libie_pci_get_mmio_addr(mmio, temp_offset); /* PHY clock offsets */ temp_offset = le32_to_cpu(clk_adj_offsets.phy_clk_cmd_type); - ptp->dev_clk_regs.phy_cmd = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.phy_cmd = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clk_adj_offsets.phy_clk_incval_l); - ptp->dev_clk_regs.phy_incval_l = idpf_get_reg_addr(adapter, - temp_offset); + ptp->dev_clk_regs.phy_incval_l = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clk_adj_offsets.phy_clk_incval_h); - ptp->dev_clk_regs.phy_incval_h = idpf_get_reg_addr(adapter, - temp_offset); + ptp->dev_clk_regs.phy_incval_h = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clk_adj_offsets.phy_clk_shadj_l); - ptp->dev_clk_regs.phy_shadj_l = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.phy_shadj_l = + libie_pci_get_mmio_addr(mmio, temp_offset); temp_offset = le32_to_cpu(clk_adj_offsets.phy_clk_shadj_h); - ptp->dev_clk_regs.phy_shadj_h = idpf_get_reg_addr(adapter, temp_offset); + ptp->dev_clk_regs.phy_shadj_h = + libie_pci_get_mmio_addr(mmio, temp_offset); - return 0; +free_resp: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + return err; } /** @@ -154,28 +160,34 @@ int idpf_ptp_get_caps(struct idpf_adapter *adapter) int idpf_ptp_get_dev_clk_time(struct idpf_adapter *adapter, struct idpf_ptp_dev_timers *dev_clk_time) { + struct virtchnl2_ptp_get_dev_clk_time *get_dev_clk_time_resp; struct virtchnl2_ptp_get_dev_clk_time get_dev_clk_time_msg; - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME, - .send_buf.iov_base = &get_dev_clk_time_msg, - .send_buf.iov_len = sizeof(get_dev_clk_time_msg), - .recv_buf.iov_base = &get_dev_clk_time_msg, - .recv_buf.iov_len = sizeof(get_dev_clk_time_msg), + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME, .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, }; - int reply_sz; + size_t reply_sz; u64 dev_time; + int err; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (reply_sz != sizeof(get_dev_clk_time_msg)) - return -EIO; + err = idpf_send_mb_msg_stack(adapter, &xn_params, + &get_dev_clk_time_msg); + if (err) + return err; - dev_time = le64_to_cpu(get_dev_clk_time_msg.dev_time_ns); + reply_sz = xn_params.recv_mem.iov_len; + if (reply_sz != sizeof(*get_dev_clk_time_resp)) { + err = -EIO; + goto free_resp; + } + + get_dev_clk_time_resp = xn_params.recv_mem.iov_base; + dev_time = le64_to_cpu(get_dev_clk_time_resp->dev_time_ns); dev_clk_time->dev_clk_time_ns = dev_time; - return 0; +free_resp: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + return err; } /** @@ -191,27 +203,29 @@ int idpf_ptp_get_dev_clk_time(struct idpf_adapter *adapter, int idpf_ptp_get_cross_time(struct idpf_adapter *adapter, struct idpf_ptp_dev_timers *cross_time) { - struct virtchnl2_ptp_get_cross_time cross_time_msg; - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_PTP_GET_CROSS_TIME, - .send_buf.iov_base = &cross_time_msg, - .send_buf.iov_len = sizeof(cross_time_msg), - .recv_buf.iov_base = &cross_time_msg, - .recv_buf.iov_len = sizeof(cross_time_msg), + struct virtchnl2_ptp_get_cross_time cross_time_send, *cross_time_recv; + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_PTP_GET_CROSS_TIME, .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, }; - int reply_sz; + int err = 0; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (reply_sz != sizeof(cross_time_msg)) - return -EIO; + err = idpf_send_mb_msg_stack(adapter, &xn_params, &cross_time_send); + if (err) + return err; - cross_time->dev_clk_time_ns = le64_to_cpu(cross_time_msg.dev_time_ns); - cross_time->sys_time_ns = le64_to_cpu(cross_time_msg.sys_time_ns); + if (xn_params.recv_mem.iov_len != sizeof(*cross_time_recv)) { + err = -EIO; + goto free_resp; + } - return 0; + cross_time_recv = xn_params.recv_mem.iov_base; + cross_time->dev_clk_time_ns = le64_to_cpu(cross_time_recv->dev_time_ns); + cross_time->sys_time_ns = le64_to_cpu(cross_time_recv->sys_time_ns); + +free_resp: + libie_ctlq_release_rx_buf(&xn_params.recv_mem); + return err; } /** @@ -228,23 +242,18 @@ int idpf_ptp_set_dev_clk_time(struct idpf_adapter *adapter, u64 time) struct virtchnl2_ptp_set_dev_clk_time set_dev_clk_time_msg = { .dev_time_ns = cpu_to_le64(time), }; - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME, - .send_buf.iov_base = &set_dev_clk_time_msg, - .send_buf.iov_len = sizeof(set_dev_clk_time_msg), - .recv_buf.iov_base = &set_dev_clk_time_msg, - .recv_buf.iov_len = sizeof(set_dev_clk_time_msg), + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME, .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, }; - int reply_sz; + int err; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (reply_sz != sizeof(set_dev_clk_time_msg)) - return -EIO; + err = idpf_send_mb_msg_stack(adapter, &xn_params, + &set_dev_clk_time_msg); + if (!err) + libie_ctlq_release_rx_buf(&xn_params.recv_mem); - return 0; + return err; } /** @@ -261,23 +270,18 @@ int idpf_ptp_adj_dev_clk_time(struct idpf_adapter *adapter, s64 delta) struct virtchnl2_ptp_adj_dev_clk_time adj_dev_clk_time_msg = { .delta = cpu_to_le64(delta), }; - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_TIME, - .send_buf.iov_base = &adj_dev_clk_time_msg, - .send_buf.iov_len = sizeof(adj_dev_clk_time_msg), - .recv_buf.iov_base = &adj_dev_clk_time_msg, - .recv_buf.iov_len = sizeof(adj_dev_clk_time_msg), + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_TIME, .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, }; - int reply_sz; + int err; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (reply_sz != sizeof(adj_dev_clk_time_msg)) - return -EIO; + err = idpf_send_mb_msg_stack(adapter, &xn_params, + &adj_dev_clk_time_msg); + if (!err) + libie_ctlq_release_rx_buf(&xn_params.recv_mem); - return 0; + return err; } /** @@ -295,23 +299,18 @@ int idpf_ptp_adj_dev_clk_fine(struct idpf_adapter *adapter, u64 incval) struct virtchnl2_ptp_adj_dev_clk_fine adj_dev_clk_fine_msg = { .incval = cpu_to_le64(incval), }; - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_FINE, - .send_buf.iov_base = &adj_dev_clk_fine_msg, - .send_buf.iov_len = sizeof(adj_dev_clk_fine_msg), - .recv_buf.iov_base = &adj_dev_clk_fine_msg, - .recv_buf.iov_len = sizeof(adj_dev_clk_fine_msg), + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_FINE, .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, }; - int reply_sz; + int err; - reply_sz = idpf_vc_xn_exec(adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; - if (reply_sz != sizeof(adj_dev_clk_fine_msg)) - return -EIO; + err = idpf_send_mb_msg_stack(adapter, &xn_params, + &adj_dev_clk_fine_msg); + if (!err) + libie_ctlq_release_rx_buf(&xn_params.recv_mem); - return 0; + return err; } /** @@ -330,18 +329,16 @@ int idpf_ptp_get_vport_tstamps_caps(struct idpf_vport *vport) struct virtchnl2_ptp_tx_tstamp_latch_caps tx_tstamp_latch_caps; struct idpf_ptp_vport_tx_tstamp_caps *tstamp_caps; struct idpf_ptp_tx_tstamp *ptp_tx_tstamp, *tmp; - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP_CAPS, - .send_buf.iov_base = &send_tx_tstamp_caps, - .send_buf.iov_len = sizeof(send_tx_tstamp_caps), - .recv_buf.iov_len = IDPF_CTLQ_MAX_BUF_LEN, + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP_CAPS, .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, }; enum idpf_ptp_access tstamp_access, get_dev_clk_access; struct idpf_ptp *ptp = vport->adapter->ptp; struct list_head *head; - int err = 0, reply_sz; + size_t reply_sz; u16 num_latches; + int err = 0; u32 size; if (!ptp) @@ -353,19 +350,19 @@ int idpf_ptp_get_vport_tstamps_caps(struct idpf_vport *vport) get_dev_clk_access == IDPF_PTP_NONE) return -EOPNOTSUPP; - rcv_tx_tstamp_caps = kzalloc(IDPF_CTLQ_MAX_BUF_LEN, GFP_KERNEL); - if (!rcv_tx_tstamp_caps) - return -ENOMEM; - send_tx_tstamp_caps.vport_id = cpu_to_le32(vport->vport_id); - xn_params.recv_buf.iov_base = rcv_tx_tstamp_caps; - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) { - err = reply_sz; + err = idpf_send_mb_msg_stack(vport->adapter, &xn_params, + &send_tx_tstamp_caps); + if (err) + return err; + + rcv_tx_tstamp_caps = xn_params.recv_mem.iov_base; + reply_sz = xn_params.recv_mem.iov_len; + if (reply_sz < sizeof(*rcv_tx_tstamp_caps)) { + err = -EIO; goto get_tstamp_caps_out; } - num_latches = le16_to_cpu(rcv_tx_tstamp_caps->num_latches); size = struct_size(rcv_tx_tstamp_caps, tstamp_latches, num_latches); if (reply_sz != size) { @@ -420,7 +417,7 @@ int idpf_ptp_get_vport_tstamps_caps(struct idpf_vport *vport) } vport->tx_tstamp_caps = tstamp_caps; - kfree(rcv_tx_tstamp_caps); + libie_ctlq_release_rx_buf(&xn_params.recv_mem); return 0; @@ -433,7 +430,7 @@ int idpf_ptp_get_vport_tstamps_caps(struct idpf_vport *vport) kfree(tstamp_caps); get_tstamp_caps_out: - kfree(rcv_tx_tstamp_caps); + libie_ctlq_release_rx_buf(&xn_params.recv_mem); return err; } @@ -530,9 +527,9 @@ idpf_ptp_get_tstamp_value(struct idpf_vport *vport, /** * idpf_ptp_get_tx_tstamp_async_handler - Async callback for getting Tx tstamps - * @adapter: Driver specific private structure - * @xn: transaction for message - * @ctlq_msg: received message + * @ctx: adapter pointer + * @mem: address and size of the response + * @status: return value of the request * * Read the tstamps Tx tstamp values from a received message and put them * directly to the skb. The number of timestamps to read is specified by @@ -540,22 +537,26 @@ idpf_ptp_get_tstamp_value(struct idpf_vport *vport, * * Return: 0 on success, -errno otherwise. */ -static int -idpf_ptp_get_tx_tstamp_async_handler(struct idpf_adapter *adapter, - struct idpf_vc_xn *xn, - const struct idpf_ctlq_msg *ctlq_msg) +static void +idpf_ptp_get_tx_tstamp_async_handler(void *ctx, struct kvec *mem, int status) { struct virtchnl2_ptp_get_vport_tx_tstamp_latches *recv_tx_tstamp_msg; struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps; struct virtchnl2_ptp_tx_tstamp_latch tstamp_latch; struct idpf_ptp_tx_tstamp *tx_tstamp, *tmp; struct idpf_vport *tstamp_vport = NULL; + struct idpf_adapter *adapter = ctx; struct list_head *head; u16 num_latches; u32 vport_id; - int err = 0; - recv_tx_tstamp_msg = ctlq_msg->ctx.indirect.payload->va; + if (status) + return; + + recv_tx_tstamp_msg = mem->iov_base; + if (mem->iov_len < sizeof(*recv_tx_tstamp_msg)) + return; + vport_id = le32_to_cpu(recv_tx_tstamp_msg->vport_id); idpf_for_each_vport(adapter, vport) { @@ -569,10 +570,13 @@ idpf_ptp_get_tx_tstamp_async_handler(struct idpf_adapter *adapter, } if (!tstamp_vport || !tstamp_vport->tx_tstamp_caps) - return -EINVAL; + return; tx_tstamp_caps = tstamp_vport->tx_tstamp_caps; num_latches = le16_to_cpu(recv_tx_tstamp_msg->num_latches); + if (mem->iov_len < struct_size(recv_tx_tstamp_msg, tstamp_latches, + num_latches)) + return; spin_lock_bh(&tx_tstamp_caps->latches_lock); head = &tx_tstamp_caps->latches_in_use; @@ -583,13 +587,13 @@ idpf_ptp_get_tx_tstamp_async_handler(struct idpf_adapter *adapter, if (!tstamp_latch.valid) continue; - if (list_empty(head)) { - err = -ENOBUFS; + if (list_empty(head)) goto unlock; - } list_for_each_entry_safe(tx_tstamp, tmp, head, list_member) { if (tstamp_latch.index == tx_tstamp->idx) { + int err; + list_del(&tx_tstamp->list_member); err = idpf_ptp_get_tstamp_value(tstamp_vport, &tstamp_latch, @@ -604,8 +608,6 @@ idpf_ptp_get_tx_tstamp_async_handler(struct idpf_adapter *adapter, unlock: spin_unlock_bh(&tx_tstamp_caps->latches_lock); - - return err; } /** @@ -621,15 +623,15 @@ int idpf_ptp_get_tx_tstamp(struct idpf_vport *vport) { struct virtchnl2_ptp_get_vport_tx_tstamp_latches *send_tx_tstamp_msg; struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps; - struct idpf_vc_xn_params xn_params = { - .vc_op = VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP, + struct libie_ctlq_xn_send_params xn_params = { + .chnl_opcode = VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP, .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, - .async = true, - .async_handler = idpf_ptp_get_tx_tstamp_async_handler, + .resp_cb = idpf_ptp_get_tx_tstamp_async_handler, + .send_ctx = vport->adapter, }; struct idpf_ptp_tx_tstamp *ptp_tx_tstamp; - int reply_sz, size, msg_size; struct list_head *head; + int size, msg_size; bool state_upd; u16 id = 0; @@ -662,11 +664,7 @@ int idpf_ptp_get_tx_tstamp(struct idpf_vport *vport) msg_size = struct_size(send_tx_tstamp_msg, tstamp_latches, id); send_tx_tstamp_msg->vport_id = cpu_to_le32(vport->vport_id); send_tx_tstamp_msg->num_latches = cpu_to_le16(id); - xn_params.send_buf.iov_base = send_tx_tstamp_msg; - xn_params.send_buf.iov_len = msg_size; - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - kfree(send_tx_tstamp_msg); - - return min(reply_sz, 0); + return idpf_send_mb_msg_kfree(vport->adapter, &xn_params, + send_tx_tstamp_msg, msg_size); } diff --git a/drivers/net/ethernet/intel/ixd/Kconfig b/drivers/net/ethernet/intel/ixd/Kconfig new file mode 100644 index 000000000000..0a48b3bb7bc2 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2025 Intel Corporation + +config IXD + tristate "Intel(R) Control Plane Function Support" + depends on PCI_MSI + select LIBIE_CP + select LIBIE_PCI + select NET_DEVLINK + help + This driver supports Intel(R) Control Plane PCI Function + of Intel E2100 and later IPUs and FNICs. + It facilitates a centralized control over multiple IDPF PFs/VFs/SFs + exposed by the same card. diff --git a/drivers/net/ethernet/intel/ixd/Makefile b/drivers/net/ethernet/intel/ixd/Makefile new file mode 100644 index 000000000000..03760a2580b9 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2025 Intel Corporation + +# Intel(R) Control Plane Function Linux Driver + +obj-$(CONFIG_IXD) += ixd.o + +ixd-y := ixd_main.o +ixd-y += ixd_ctlq.o +ixd-y += ixd_dev.o +ixd-y += ixd_devlink.o +ixd-y += ixd_lib.o +ixd-y += ixd_virtchnl.o diff --git a/drivers/net/ethernet/intel/ixd/ixd.h b/drivers/net/ethernet/intel/ixd/ixd.h new file mode 100644 index 000000000000..2a09ccba13d5 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef _IXD_H_ +#define _IXD_H_ + +#include + +#define IXD_INIT_TASK_DELAY_JIFFIES msecs_to_jiffies(500) + +/** + * struct ixd_adapter - Data structure representing a CPF + * @cp_ctx: Control plane communication context + * @init_task: Delayed initialization after reset + * @init_task.init_work: Delayed initialization work + * @init_task.reset_retries: How many times to check, whether reset is completed + * @init_task.vc_retries: Number of retries to establish mailbox communication + * @init_task.success: init_work completion status + * @mbx_task: Control queue Rx handling + * @xnm: virtchnl transaction manager + * @asq: Send control queue info + * @arq: Receive control queue info + * @vc_ver: Negotiated virtchnl version + * @vc_ver.major: Negotiated major virtchnl version + * @vc_ver.minor: Negotiated minor virtchnl version + * @caps: Negotiated virtchnl capabilities + */ +struct ixd_adapter { + struct libie_ctlq_ctx cp_ctx; + struct { + struct delayed_work init_work; + u8 reset_retries; + u8 vc_retries; + bool success; + } init_task; + struct delayed_work mbx_task; + struct libie_ctlq_xn_manager *xnm; + struct libie_ctlq_info *asq; + struct libie_ctlq_info *arq; + struct { + u32 major; + u32 minor; + } vc_ver; + struct virtchnl2_get_capabilities caps; +}; + +/** + * ixd_to_dev - Get the corresponding device struct from an adapter + * @adapter: PCI device driver-specific private data + * + * Return: struct device corresponding to the given adapter + */ +static inline struct device *ixd_to_dev(struct ixd_adapter *adapter) +{ + return &adapter->cp_ctx.mmio_info.pdev->dev; +} + +void ixd_ctlq_reg_init(struct ixd_adapter *adapter, + struct libie_ctlq_reg *ctlq_reg_tx, + struct libie_ctlq_reg *ctlq_reg_rx); +void ixd_trigger_reset(struct ixd_adapter *adapter); +bool ixd_check_reset_complete(struct ixd_adapter *adapter); +void ixd_init_task(struct work_struct *work); +int ixd_init_dflt_mbx(struct ixd_adapter *adapter); +void ixd_deinit_dflt_mbx(struct ixd_adapter *adapter); + +#endif /* _IXD_H_ */ diff --git a/drivers/net/ethernet/intel/ixd/ixd_ctlq.c b/drivers/net/ethernet/intel/ixd/ixd_ctlq.c new file mode 100644 index 000000000000..8712e10c8c50 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_ctlq.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include "ixd.h" +#include "ixd_ctlq.h" +#include "ixd_virtchnl.h" + +#define IXD_CTLQ_RX_TASK_DELAY_JIFFIES msecs_to_jiffies(300) + +/** + * ixd_ctlq_clean_sq - Clean the send control queue after sending the message + * @adapter: The adapter that sent the messages + * @force: Clean regardless of send status + * + * Free the libie send resources after sending the message and handling + * the response. + */ +void ixd_ctlq_clean_sq(struct ixd_adapter *adapter, bool force) +{ + libie_ctlq_xn_send_clean(adapter->asq, kfree, force); +} + +/** + * ixd_ctlq_init_sparams - Initialize control queue send parameters + * @adapter: The adapter with initialized mailbox + * @sparams: Parameters to initialize + * @msg_buf: DMA-mappable pointer to the message being sent + * @msg_size: Message size + */ +static void ixd_ctlq_init_sparams(struct ixd_adapter *adapter, + struct libie_ctlq_xn_send_params *sparams, + void *msg_buf, size_t msg_size) +{ + *sparams = (struct libie_ctlq_xn_send_params) { + .rel_tx_buf = kfree, + .xnm = adapter->xnm, + .ctlq = adapter->asq, + .timeout_ms = IXD_CTLQ_TIMEOUT, + .send_buf = (struct kvec) { + .iov_base = msg_buf, + .iov_len = msg_size, + }, + }; +} + +/** + * ixd_ctlq_do_req - Perform a standard virtchnl request + * @adapter: The adapter with initialized mailbox + * @req: virtchnl request description + * + * Return: %0 if a message was sent and received a response + * that was successfully handled by the custom callback, + * negative error otherwise. + */ +int ixd_ctlq_do_req(struct ixd_adapter *adapter, const struct ixd_ctlq_req *req) +{ + u8 onstack_send_buff[LIBIE_CP_TX_COPYBREAK] __aligned_largest = {}; + struct libie_ctlq_xn_send_params send_params = {}; + struct kvec *recv_mem; + void *send_buff; + int err; + + send_buff = libie_cp_can_send_onstack(req->send_size) ? + &onstack_send_buff : kzalloc(req->send_size, GFP_KERNEL); + if (!send_buff) + return -ENOMEM; + + ixd_ctlq_init_sparams(adapter, &send_params, send_buff, + req->send_size); + + send_params.chnl_opcode = req->opcode; + + if (req->send_buff_init) + req->send_buff_init(adapter, send_buff, req->ctx); + + ixd_ctlq_clean_sq(adapter, false); + err = libie_ctlq_xn_send(&send_params); + if (err) + return err; + + recv_mem = &send_params.recv_mem; + if (req->recv_process) + err = req->recv_process(adapter, recv_mem->iov_base, + recv_mem->iov_len, req->ctx); + + libie_ctlq_release_rx_buf(recv_mem); + + return err; +} + +/** + * ixd_ctlq_handle_msg - Default control queue message handler + * @ctx: Control plane communication context + * @msg: Message received + */ +static void ixd_ctlq_handle_msg(struct libie_ctlq_ctx *ctx, + struct libie_ctlq_msg *msg) +{ + struct ixd_adapter *adapter = pci_get_drvdata(ctx->mmio_info.pdev); + + if (ixd_vc_can_handle_msg(msg)) + ixd_vc_recv_event_msg(adapter, msg); + else + dev_dbg_ratelimited(ixd_to_dev(adapter), + "Received an unsupported opcode 0x%x from the CP\n", + msg->chnl_opcode); + + libie_ctlq_release_rx_buf(&msg->recv_mem); +} + +/** + * ixd_ctlq_recv_mb_msg - Receive a potential message over mailbox periodically + * @adapter: The adapter with initialized mailbox + */ +static void ixd_ctlq_recv_mb_msg(struct ixd_adapter *adapter) +{ + struct libie_ctlq_xn_recv_params xn_params = { + .xnm = adapter->xnm, + .ctlq = adapter->arq, + .ctlq_msg_handler = ixd_ctlq_handle_msg, + .budget = LIBIE_CTLQ_MAX_XN_ENTRIES, + }; + + libie_ctlq_xn_recv(&xn_params); +} + +/** + * ixd_ctlq_rx_task - Periodically check for mailbox responses and events + * @work: work handle + */ +void ixd_ctlq_rx_task(struct work_struct *work) +{ + struct ixd_adapter *adapter; + + adapter = container_of(work, struct ixd_adapter, mbx_task.work); + + queue_delayed_work(system_dfl_wq, &adapter->mbx_task, + IXD_CTLQ_RX_TASK_DELAY_JIFFIES); + + ixd_ctlq_recv_mb_msg(adapter); +} diff --git a/drivers/net/ethernet/intel/ixd/ixd_ctlq.h b/drivers/net/ethernet/intel/ixd/ixd_ctlq.h new file mode 100644 index 000000000000..8839f9f8f6d5 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_ctlq.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef _IXD_CTLQ_H_ +#define _IXD_CTLQ_H_ + +#include + +#define IXD_CTLQ_TIMEOUT 2000 + +/** + * struct ixd_ctlq_req - Standard virtchnl request description + * @opcode: protocol opcode, only virtchnl2 is needed for now + * @send_size: required length of the send buffer + * @send_buff_init: function to initialize the allocated send buffer + * @recv_process: function to handle the CP response + * @ctx: additional context for callbacks + */ +struct ixd_ctlq_req { + enum virtchnl2_op opcode; + size_t send_size; + void (*send_buff_init)(struct ixd_adapter *adapter, void *send_buff, + void *ctx); + int (*recv_process)(struct ixd_adapter *adapter, void *recv_buff, + size_t recv_size, void *ctx); + void *ctx; +}; + +void ixd_ctlq_clean_sq(struct ixd_adapter *adapter, bool force); +int ixd_ctlq_do_req(struct ixd_adapter *adapter, + const struct ixd_ctlq_req *req); +void ixd_ctlq_rx_task(struct work_struct *work); + +#endif /* _IXD_CTLQ_H_ */ diff --git a/drivers/net/ethernet/intel/ixd/ixd_dev.c b/drivers/net/ethernet/intel/ixd/ixd_dev.c new file mode 100644 index 000000000000..cdd5477cc1f4 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_dev.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include "ixd.h" +#include "ixd_lan_regs.h" + +/** + * ixd_ctlq_reg_init - Initialize default mailbox registers + * @adapter: PCI device driver-specific private data + * @ctlq_reg_tx: Transmit queue registers info to be filled + * @ctlq_reg_rx: Receive queue registers info to be filled + */ +void ixd_ctlq_reg_init(struct ixd_adapter *adapter, + struct libie_ctlq_reg *ctlq_reg_tx, + struct libie_ctlq_reg *ctlq_reg_rx) +{ + struct libie_mmio_info *mmio_info = &adapter->cp_ctx.mmio_info; + *ctlq_reg_tx = (struct libie_ctlq_reg) { + .head = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQH), + .tail = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQT), + .len = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQLEN), + .addr_high = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQBAH), + .addr_low = libie_pci_get_mmio_addr(mmio_info, PF_FW_ATQBAL), + .len_mask = PF_FW_ATQLEN_ATQLEN_M, + .len_ena_mask = PF_FW_ATQLEN_ATQENABLE_M, + .head_mask = PF_FW_ATQH_ATQH_M, + }; + + *ctlq_reg_rx = (struct libie_ctlq_reg) { + .head = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQH), + .tail = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQT), + .len = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQLEN), + .addr_high = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQBAH), + .addr_low = libie_pci_get_mmio_addr(mmio_info, PF_FW_ARQBAL), + .len_mask = PF_FW_ARQLEN_ARQLEN_M, + .len_ena_mask = PF_FW_ARQLEN_ARQENABLE_M, + .head_mask = PF_FW_ARQH_ARQH_M, + }; +} + +static const struct ixd_reset_reg ixd_reset_reg = { + .rstat = PFGEN_RSTAT, + .rstat_m = PFGEN_RSTAT_PFR_STATE_M, + .rstat_ok_v = 0b01, + .rtrigger = PFGEN_CTRL, + .rtrigger_m = PFGEN_CTRL_PFSWR, +}; + +/** + * ixd_trigger_reset - Trigger PFR reset + * @adapter: the device with mapped reset register + */ +void ixd_trigger_reset(struct ixd_adapter *adapter) +{ + void __iomem *addr; + u32 reg_val; + + addr = libie_pci_get_mmio_addr(&adapter->cp_ctx.mmio_info, + ixd_reset_reg.rtrigger); + reg_val = readl(addr); + writel(reg_val | ixd_reset_reg.rtrigger_m, addr); +} + +/** + * ixd_check_reset_complete - Check if the PFR reset is completed + * @adapter: CPF being reset + * + * Return: %true if the register read indicates reset has been finished, + * %false otherwise + */ +bool ixd_check_reset_complete(struct ixd_adapter *adapter) +{ + u32 reg_val, reset_status; + void __iomem *addr; + + addr = libie_pci_get_mmio_addr(&adapter->cp_ctx.mmio_info, + ixd_reset_reg.rstat); + reg_val = readl(addr); + reset_status = reg_val & ixd_reset_reg.rstat_m; + + /* 0xFFFFFFFF might be read if the other side hasn't cleared + * the register for us yet. + */ + if (reg_val != GENMASK(31, 0) && + reset_status == ixd_reset_reg.rstat_ok_v) + return true; + + return false; +} diff --git a/drivers/net/ethernet/intel/ixd/ixd_devlink.c b/drivers/net/ethernet/intel/ixd/ixd_devlink.c new file mode 100644 index 000000000000..828132db8323 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_devlink.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2025, Intel Corporation. */ + +#include "ixd.h" +#include "ixd_devlink.h" + +#define IXD_DEVLINK_INFO_LEN 128 + +/** + * ixd_fill_dsn - Get the serial number for the ixd device + * @adapter: adapter to query + * @buf: storage buffer for the info request + */ +static void ixd_fill_dsn(struct ixd_adapter *adapter, char *buf) +{ + u8 dsn[8]; + + /* Copy the DSN into an array in Big Endian format */ + put_unaligned_be64(pci_get_dsn(adapter->cp_ctx.mmio_info.pdev), dsn); + + snprintf(buf, IXD_DEVLINK_INFO_LEN, "%8phD", dsn); +} + +/** + * ixd_fill_device_name - Get the name of the underlying hardware + * @adapter: adapter to query + * @buf: storage buffer for the info request + * @buf_size: size of the storage buffer + */ +static void ixd_fill_device_name(struct ixd_adapter *adapter, char *buf, + size_t buf_size) +{ + if (adapter->caps.device_type == cpu_to_le32(VIRTCHNL2_MEV_DEVICE)) + snprintf(buf, buf_size, "%s", "MEV"); + else + snprintf(buf, buf_size, "%s", "UNKNOWN"); +} + +/** + * ixd_devlink_info_get - .info_get devlink handler + * @devlink: devlink instance structure + * @req: the devlink info request + * @extack: extended netdev ack structure + * + * Callback for the devlink .info_get operation. Reports information about the + * device. + * + * Return: zero on success or an error code on failure. + */ +static int ixd_devlink_info_get(struct devlink *devlink, + struct devlink_info_req *req, + struct netlink_ext_ack *extack) +{ + struct ixd_adapter *adapter = devlink_priv(devlink); + char buf[IXD_DEVLINK_INFO_LEN]; + int err; + + ixd_fill_dsn(adapter, buf); + err = devlink_info_serial_number_put(req, buf); + if (err) + return err; + + ixd_fill_device_name(adapter, buf, IXD_DEVLINK_INFO_LEN); + err = devlink_info_version_fixed_put(req, "device.type", buf); + if (err) + return err; + + snprintf(buf, sizeof(buf), "%u.%u", + adapter->vc_ver.major, adapter->vc_ver.minor); + + return devlink_info_version_running_put(req, + DEVLINK_INFO_VERSION_GENERIC_FW_MGMT_API, + buf); +} + +static const struct devlink_ops ixd_devlink_ops = { + .info_get = ixd_devlink_info_get, +}; + +/** + * ixd_adapter_alloc - Allocate devlink and return adapter pointer + * @dev: the device to allocate for + * + * Allocate a devlink instance for this device and return the private area as + * the adapter structure. + * + * Return: adapter structure on success, NULL on failure + */ +struct ixd_adapter *ixd_adapter_alloc(struct device *dev) +{ + struct devlink *devlink; + + devlink = devlink_alloc(&ixd_devlink_ops, sizeof(struct ixd_adapter), + dev); + if (!devlink) + return NULL; + + return devlink_priv(devlink); +} diff --git a/drivers/net/ethernet/intel/ixd/ixd_devlink.h b/drivers/net/ethernet/intel/ixd/ixd_devlink.h new file mode 100644 index 000000000000..b23a1b37aebc --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_devlink.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2025, Intel Corporation. */ + +#ifndef _IXD_DEVLINK_H_ +#define _IXD_DEVLINK_H_ + +#include + +#include "ixd.h" + +struct ixd_adapter *ixd_adapter_alloc(struct device *dev); + +/** + * ixd_devlink_free - teardown the devlink + * @adapter: the adapter structure to free + * + */ +static inline void ixd_devlink_free(struct ixd_adapter *adapter) +{ + struct devlink *devlink = priv_to_devlink(adapter); + + devlink_free(devlink); +} + +/** + * ixd_devlink_unregister - Unregister devlink for this adapter. + * @adapter: the adapter structure to cleanup + * + * Init task must be completed or cancelled beforehand. + */ +static inline void ixd_devlink_unregister(struct ixd_adapter *adapter) +{ + if (!adapter->init_task.success) + return; + + devlink_unregister(priv_to_devlink(adapter)); +} + +/** + * ixd_devlink_register - Register devlink interface for this adapter + * @adapter: pointer to ixd adapter structure to be associated with devlink + * + * Register the devlink instance associated with this adapter + */ +static inline void ixd_devlink_register(struct ixd_adapter *adapter) +{ + devlink_register(priv_to_devlink(adapter)); +} + +#endif /* _IXD_DEVLINK_H_ */ diff --git a/drivers/net/ethernet/intel/ixd/ixd_lan_regs.h b/drivers/net/ethernet/intel/ixd/ixd_lan_regs.h new file mode 100644 index 000000000000..58e58c75981b --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_lan_regs.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef _IXD_LAN_REGS_H_ +#define _IXD_LAN_REGS_H_ + +/* Control Plane Function PCI ID */ +#define IXD_DEV_ID_CPF 0x1efe + +/* Control Queue (Mailbox) */ +#define PF_FW_MBX_REG_LEN 4096 +#define PF_FW_MBX 0x08400000 + +#define PF_FW_ARQBAL (PF_FW_MBX) +#define PF_FW_ARQBAH (PF_FW_MBX + 0x4) +#define PF_FW_ARQLEN (PF_FW_MBX + 0x8) +#define PF_FW_ARQLEN_ARQLEN_M GENMASK(12, 0) +#define PF_FW_ARQLEN_ARQENABLE_S 31 +#define PF_FW_ARQLEN_ARQENABLE_M BIT(PF_FW_ARQLEN_ARQENABLE_S) +#define PF_FW_ARQH_ARQH_M GENMASK(12, 0) +#define PF_FW_ARQH (PF_FW_MBX + 0xC) +#define PF_FW_ARQT (PF_FW_MBX + 0x10) + +#define PF_FW_ATQBAL (PF_FW_MBX + 0x14) +#define PF_FW_ATQBAH (PF_FW_MBX + 0x18) +#define PF_FW_ATQLEN (PF_FW_MBX + 0x1C) +#define PF_FW_ATQLEN_ATQLEN_M GENMASK(9, 0) +#define PF_FW_ATQLEN_ATQENABLE_S 31 +#define PF_FW_ATQLEN_ATQENABLE_M BIT(PF_FW_ATQLEN_ATQENABLE_S) +#define PF_FW_ATQH_ATQH_M GENMASK(9, 0) +#define PF_FW_ATQH (PF_FW_MBX + 0x20) +#define PF_FW_ATQT (PF_FW_MBX + 0x24) + +/* Reset registers */ +#define PFGEN_RTRIG_REG_LEN 2048 +#define PFGEN_RTRIG 0x08407000 /* Device resets */ +#define PFGEN_RSTAT 0x08407008 /* PFR status */ +#define PFGEN_RSTAT_PFR_STATE_M GENMASK(1, 0) +#define PFGEN_CTRL 0x0840700C /* PFR trigger */ +#define PFGEN_CTRL_PFSWR BIT(0) + +/** + * struct ixd_bar_region - BAR region description + * @offset: BAR region offset + * @size: BAR region size + */ +struct ixd_bar_region { + resource_size_t offset; + resource_size_t size; +}; + +/** + * struct ixd_reset_reg - structure for reset registers + * @rstat: offset of status in register + * @rstat_m: status mask + * @rstat_ok_v: value that indicates PFR completed status + * @rtrigger: offset of reset trigger in register + * @rtrigger_m: reset trigger mask + */ +struct ixd_reset_reg { + u32 rstat; + u32 rstat_m; + u32 rstat_ok_v; + u32 rtrigger; + u32 rtrigger_m; +}; + +#endif /* _IXD_LAN_REGS_H_ */ diff --git a/drivers/net/ethernet/intel/ixd/ixd_lib.c b/drivers/net/ethernet/intel/ixd/ixd_lib.c new file mode 100644 index 000000000000..8311f7590666 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_lib.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include "ixd.h" +#include "ixd_ctlq.h" +#include "ixd_devlink.h" +#include "ixd_virtchnl.h" + +#define IXD_DFLT_MBX_Q_LEN 64 + +/** + * ixd_init_ctlq_create_info - Initialize control queue info for creation + * @info: destination + * @type: type of the queue to create + * @ctlq_reg: register assigned to the control queue + */ +static void ixd_init_ctlq_create_info(struct libie_ctlq_create_info *info, + enum libie_ctlq_type type, + const struct libie_ctlq_reg *ctlq_reg) +{ + *info = (struct libie_ctlq_create_info) { + .type = type, + .id = -1, + .reg = *ctlq_reg, + .len = IXD_DFLT_MBX_Q_LEN, + }; +} + +/** + * ixd_init_libie_xn_params - Initialize xn transaction manager creation info + * @params: destination + * @adapter: adapter info struct + * @ctlqs: list of the managed queues to create + * @num_queues: length of the queue list + */ +static void ixd_init_libie_xn_params(struct libie_ctlq_xn_init_params *params, + struct ixd_adapter *adapter, + struct libie_ctlq_create_info *ctlqs, + uint num_queues) +{ + *params = (struct libie_ctlq_xn_init_params){ + .cctlq_info = ctlqs, + .ctx = &adapter->cp_ctx, + .num_qs = num_queues, + }; +} + +/** + * ixd_adapter_fill_dflt_ctlqs - Find default control queues and store them + * @adapter: adapter info struct + */ +static void ixd_adapter_fill_dflt_ctlqs(struct ixd_adapter *adapter) +{ + adapter->arq = libie_find_ctlq(&adapter->cp_ctx, LIBIE_CTLQ_TYPE_RX, + LIBIE_CTLQ_MBX_ID); + adapter->asq = libie_find_ctlq(&adapter->cp_ctx, LIBIE_CTLQ_TYPE_TX, + LIBIE_CTLQ_MBX_ID); +} + +/** + * ixd_deinit_dflt_mbx - Deinitialize default mailbox + * @adapter: adapter info struct + */ +void ixd_deinit_dflt_mbx(struct ixd_adapter *adapter) +{ + cancel_delayed_work_sync(&adapter->mbx_task); + + if (adapter->xnm) + libie_ctlq_xn_shutdown(adapter->xnm); + + if (adapter->asq) + ixd_ctlq_clean_sq(adapter, true); + + if (adapter->xnm) + libie_ctlq_xn_deinit(adapter->xnm, &adapter->cp_ctx); + + adapter->arq = NULL; + adapter->asq = NULL; + adapter->xnm = NULL; +} + +/** + * ixd_init_dflt_mbx - Setup default mailbox parameters and make request + * @adapter: adapter info struct + * + * Return: %0 on success, negative errno code on failure + */ +int ixd_init_dflt_mbx(struct ixd_adapter *adapter) +{ + struct libie_ctlq_create_info ctlqs_info[2]; + struct libie_ctlq_xn_init_params xn_params; + struct libie_ctlq_reg ctlq_reg_tx; + struct libie_ctlq_reg ctlq_reg_rx; + int err; + + ixd_ctlq_reg_init(adapter, &ctlq_reg_tx, &ctlq_reg_rx); + ixd_init_ctlq_create_info(&ctlqs_info[0], LIBIE_CTLQ_TYPE_TX, + &ctlq_reg_tx); + ixd_init_ctlq_create_info(&ctlqs_info[1], LIBIE_CTLQ_TYPE_RX, + &ctlq_reg_rx); + ixd_init_libie_xn_params(&xn_params, adapter, ctlqs_info, + ARRAY_SIZE(ctlqs_info)); + err = libie_ctlq_xn_init(&xn_params); + if (err) + return err; + adapter->xnm = xn_params.xnm; + + ixd_adapter_fill_dflt_ctlqs(adapter); + + if (!adapter->asq || !adapter->arq) { + ixd_deinit_dflt_mbx(adapter); + return -ENOENT; + } + + queue_delayed_work(system_dfl_wq, &adapter->mbx_task, 0); + + return 0; +} + +/** + * ixd_init_task - Initialize after reset + * @work: init work struct + */ +void ixd_init_task(struct work_struct *work) +{ + struct ixd_adapter *adapter; + int err; + + adapter = container_of(work, struct ixd_adapter, + init_task.init_work.work); + + if (!ixd_check_reset_complete(adapter)) { + if (++adapter->init_task.reset_retries < 10) + queue_delayed_work(system_dfl_wq, + &adapter->init_task.init_work, + IXD_INIT_TASK_DELAY_JIFFIES); + else + dev_err(ixd_to_dev(adapter), + "Device reset failed. The driver was unable to contact the device's firmware. Check that the FW is running.\n"); + return; + } + + adapter->init_task.reset_retries = 0; + err = ixd_init_dflt_mbx(adapter); + if (err) { + dev_err(ixd_to_dev(adapter), + "Failed to initialize the default mailbox: %pe\n", + ERR_PTR(err)); + return; + } + + err = ixd_vc_dev_init(adapter); + if (!err) { + adapter->init_task.vc_retries = 0; + adapter->init_task.success = true; + ixd_devlink_register(adapter); + return; + } + + libie_ctlq_xn_shutdown(adapter->xnm); + ixd_trigger_reset(adapter); + ixd_deinit_dflt_mbx(adapter); + if (++adapter->init_task.vc_retries > 5 || + (err != -ETIMEDOUT && err != -EAGAIN && err != -EBUSY)) { + dev_err(ixd_to_dev(adapter), + "Failed to establish mailbox communication with the hardware: %pe\n", + ERR_PTR(err)); + return; + } + + queue_delayed_work(system_dfl_wq, &adapter->init_task.init_work, + IXD_INIT_TASK_DELAY_JIFFIES); +} diff --git a/drivers/net/ethernet/intel/ixd/ixd_main.c b/drivers/net/ethernet/intel/ixd/ixd_main.c new file mode 100644 index 000000000000..5db992d4f9fb --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_main.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include "ixd.h" +#include "ixd_ctlq.h" +#include "ixd_lan_regs.h" +#include "ixd_devlink.h" + +MODULE_DESCRIPTION("Intel(R) Control Plane Function Device Driver"); +MODULE_IMPORT_NS("LIBIE_CP"); +MODULE_IMPORT_NS("LIBIE_PCI"); +MODULE_LICENSE("GPL"); + +/** + * ixd_remove - remove a CPF PCI device + * @pdev: PCI device being removed + */ +static void ixd_remove(struct pci_dev *pdev) +{ + struct ixd_adapter *adapter = pci_get_drvdata(pdev); + + /* Do not mix removal with (re)initialization */ + cancel_delayed_work_sync(&adapter->init_task.init_work); + + ixd_devlink_unregister(adapter); + + /* Leave the device clean on exit */ + if (adapter->xnm) + libie_ctlq_xn_shutdown(adapter->xnm); + ixd_trigger_reset(adapter); + ixd_deinit_dflt_mbx(adapter); + + libie_pci_unmap_all_mmio_regions(&adapter->cp_ctx.mmio_info); + ixd_devlink_free(adapter); +} + +/** + * ixd_shutdown - shut down a CPF PCI device + * @pdev: PCI device being shut down + */ +static void ixd_shutdown(struct pci_dev *pdev) +{ + ixd_remove(pdev); + + if (system_state == SYSTEM_POWER_OFF) + pci_set_power_state(pdev, PCI_D3hot); +} + +/** + * ixd_iomap_regions - iomap PCI BARs + * @adapter: adapter to map memory regions for + * + * Returns: %0 on success, negative on failure + */ +static int ixd_iomap_regions(struct ixd_adapter *adapter) +{ + const struct ixd_bar_region regions[] = { + { + .offset = PFGEN_RTRIG, + .size = PFGEN_RTRIG_REG_LEN, + }, + { + .offset = PF_FW_MBX, + .size = PF_FW_MBX_REG_LEN, + }, + }; + + for (int i = 0; i < ARRAY_SIZE(regions); i++) { + struct libie_mmio_info *mmio_info = &adapter->cp_ctx.mmio_info; + bool map_ok; + + map_ok = libie_pci_map_mmio_region(mmio_info, + regions[i].offset, + regions[i].size); + if (!map_ok) { + dev_err(ixd_to_dev(adapter), + "Failed to map PCI device MMIO region\n"); + + libie_pci_unmap_all_mmio_regions(mmio_info); + return -EIO; + } + } + + return 0; +} + +/** + * ixd_probe - probe a CPF PCI device + * @pdev: corresponding PCI device + * @ent: entry in ixd_pci_tbl + * + * Returns: %0 on success, negative errno code on failure + */ +static int ixd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct ixd_adapter *adapter; + int err; + + adapter = ixd_adapter_alloc(&pdev->dev); + if (!adapter) + return -ENOMEM; + + adapter->cp_ctx.mmio_info.pdev = pdev; + INIT_LIST_HEAD(&adapter->cp_ctx.mmio_info.mmio_list); + + err = libie_pci_init_dev(pdev); + if (err) + goto free_adapter; + + pci_set_drvdata(pdev, adapter); + + err = ixd_iomap_regions(adapter); + if (err) + goto free_adapter; + + INIT_DELAYED_WORK(&adapter->init_task.init_work, + ixd_init_task); + INIT_DELAYED_WORK(&adapter->mbx_task, ixd_ctlq_rx_task); + + ixd_trigger_reset(adapter); + queue_delayed_work(system_dfl_wq, &adapter->init_task.init_work, + IXD_INIT_TASK_DELAY_JIFFIES); + + return 0; + +free_adapter: + ixd_devlink_free(adapter); + return err; +} + +static const struct pci_device_id ixd_pci_tbl[] = { + { PCI_VDEVICE(INTEL, IXD_DEV_ID_CPF) }, + { } +}; +MODULE_DEVICE_TABLE(pci, ixd_pci_tbl); + +static struct pci_driver ixd_driver = { + .name = KBUILD_MODNAME, + .id_table = ixd_pci_tbl, + .probe = ixd_probe, + .remove = ixd_remove, + .shutdown = ixd_shutdown, +}; +module_pci_driver(ixd_driver); diff --git a/drivers/net/ethernet/intel/ixd/ixd_virtchnl.c b/drivers/net/ethernet/intel/ixd/ixd_virtchnl.c new file mode 100644 index 000000000000..fc3b6d2e28c5 --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_virtchnl.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include "ixd.h" +#include "ixd_ctlq.h" +#include "ixd_virtchnl.h" + +/** + * ixd_vc_recv_event_msg - Handle virtchnl event message + * @adapter: The adapter handling the message + * @ctlq_msg: Message received + */ +void ixd_vc_recv_event_msg(struct ixd_adapter *adapter, + struct libie_ctlq_msg *ctlq_msg) +{ + int payload_size = ctlq_msg->data_len; + struct virtchnl2_event *v2e; + + if (payload_size < sizeof(*v2e)) { + dev_warn_ratelimited(ixd_to_dev(adapter), + "Failed to receive valid payload for event msg (op 0x%X len %u)\n", + ctlq_msg->chnl_opcode, + payload_size); + return; + } + + v2e = (struct virtchnl2_event *)ctlq_msg->recv_mem.iov_base; + + dev_dbg(ixd_to_dev(adapter), "Got event 0x%X from the CP\n", + le32_to_cpu(v2e->event)); +} + +/** + * ixd_vc_can_handle_msg - Decide if an event has to be handled by virtchnl code + * @ctlq_msg: Message received + * + * Return: %true if virtchnl code can handle the event, %false otherwise + */ +bool ixd_vc_can_handle_msg(struct libie_ctlq_msg *ctlq_msg) +{ + return ctlq_msg->chnl_opcode == VIRTCHNL2_OP_EVENT; +} + +/** + * ixd_handle_caps - Handle VIRTCHNL2_OP_GET_CAPS response + * @adapter: The adapter for which the capabilities are being updated + * @recv_buff: Buffer containing the response + * @recv_size: Response buffer size + * @ctx: unused + * + * Return: %0 if the response format is correct and was handled as expected, + * negative error otherwise. + */ +static int ixd_handle_caps(struct ixd_adapter *adapter, void *recv_buff, + size_t recv_size, void *ctx) +{ + if (recv_size < sizeof(adapter->caps)) + return -EBADMSG; + + adapter->caps = *(typeof(adapter->caps) *)recv_buff; + + return 0; +} + +/** + * ixd_req_vc_caps - Request and save device capability + * @adapter: The adapter to get the capabilities for + * + * Return: success or error if sending the get capability message fails + */ +static int ixd_req_vc_caps(struct ixd_adapter *adapter) +{ + const struct ixd_ctlq_req req = { + .opcode = VIRTCHNL2_OP_GET_CAPS, + .send_size = sizeof(struct virtchnl2_get_capabilities), + .ctx = NULL, + .send_buff_init = NULL, + .recv_process = ixd_handle_caps, + }; + + return ixd_ctlq_do_req(adapter, &req); +} + +/** + * ixd_get_vc_ver - Get version info from adapter + * + * Return: filled in virtchannel2 version info, ready for sending + */ +static struct virtchnl2_version_info ixd_get_vc_ver(void) +{ + return (struct virtchnl2_version_info) { + .major = cpu_to_le32(VIRTCHNL2_VERSION_MAJOR_2), + .minor = cpu_to_le32(VIRTCHNL2_VERSION_MINOR_0), + }; +} + +static void ixd_fill_vc_ver(struct ixd_adapter *adapter, void *send_buff, + void *ctx) +{ + *(struct virtchnl2_version_info *)send_buff = ixd_get_vc_ver(); +} + +/** + * ixd_handle_vc_ver - Handle VIRTCHNL2_OP_VERSION response + * @adapter: The adapter for which the version is being updated + * @recv_buff: Buffer containing the response + * @recv_size: Response buffer size + * @ctx: Unused + * + * Return: %0 if the response format is correct and was handled as expected, + * negative error otherwise. + */ +static int ixd_handle_vc_ver(struct ixd_adapter *adapter, void *recv_buff, + size_t recv_size, void *ctx) +{ + struct virtchnl2_version_info need_ver = ixd_get_vc_ver(); + struct virtchnl2_version_info *recv_ver; + + if (recv_size < sizeof(need_ver)) + return -EBADMSG; + + recv_ver = recv_buff; + if (le32_to_cpu(need_ver.major) != le32_to_cpu(recv_ver->major) || + le32_to_cpu(need_ver.minor) != le32_to_cpu(recv_ver->minor)) + dev_warn(ixd_to_dev(adapter), + "Virtchnl version does not match (expected %u.%u, received %u.%u)\n", + le32_to_cpu(need_ver.major), + le32_to_cpu(need_ver.minor), + le32_to_cpu(recv_ver->major), + le32_to_cpu(recv_ver->minor)); + + if (le32_to_cpu(need_ver.major) != le32_to_cpu(recv_ver->major)) { + dev_err(ixd_to_dev(adapter), + "Device initialization failed due to virtchnl major version mismatch\n"); + return -EOPNOTSUPP; + } + + adapter->vc_ver.major = le32_to_cpu(recv_ver->major); + adapter->vc_ver.minor = le32_to_cpu(recv_ver->minor); + + return 0; +} + +/** + * ixd_req_vc_version - Request and save Virtchannel2 version + * @adapter: The adapter to get the version for + * + * Return: success or error if sending fails or the response was not as expected + */ +static int ixd_req_vc_version(struct ixd_adapter *adapter) +{ + const struct ixd_ctlq_req req = { + .opcode = VIRTCHNL2_OP_VERSION, + .send_size = sizeof(struct virtchnl2_version_info), + .ctx = NULL, + .send_buff_init = ixd_fill_vc_ver, + .recv_process = ixd_handle_vc_ver, + }; + + return ixd_ctlq_do_req(adapter, &req); +} + +/** + * ixd_vc_dev_init - virtchnl device core initialization + * @adapter: device information + * + * Return: %0 on success or error if any step of the initialization fails + */ +int ixd_vc_dev_init(struct ixd_adapter *adapter) +{ + int err; + + err = ixd_req_vc_version(adapter); + if (err) { + dev_warn(ixd_to_dev(adapter), + "Getting virtchnl version failed, error=%pe\n", + ERR_PTR(err)); + return err; + } + + err = ixd_req_vc_caps(adapter); + if (err) { + dev_warn(ixd_to_dev(adapter), + "Getting virtchnl capabilities failed, error=%pe\n", + ERR_PTR(err)); + return err; + } + + return err; +} diff --git a/drivers/net/ethernet/intel/ixd/ixd_virtchnl.h b/drivers/net/ethernet/intel/ixd/ixd_virtchnl.h new file mode 100644 index 000000000000..1a53da8b545c --- /dev/null +++ b/drivers/net/ethernet/intel/ixd/ixd_virtchnl.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef _IXD_VIRTCHNL_H_ +#define _IXD_VIRTCHNL_H_ + +int ixd_vc_dev_init(struct ixd_adapter *adapter); +bool ixd_vc_can_handle_msg(struct libie_ctlq_msg *ctlq_msg); +void ixd_vc_recv_event_msg(struct ixd_adapter *adapter, + struct libie_ctlq_msg *ctlq_msg); + +#endif /* _IXD_VIRTCHNL_H_ */ diff --git a/drivers/net/ethernet/intel/libeth/rx.c b/drivers/net/ethernet/intel/libeth/rx.c index 62521a1f4ec9..0c1a565a1b3a 100644 --- a/drivers/net/ethernet/intel/libeth/rx.c +++ b/drivers/net/ethernet/intel/libeth/rx.c @@ -145,25 +145,29 @@ static bool libeth_rx_page_pool_params_zc(struct libeth_fq *fq, /** * libeth_rx_fq_create - create a PP with the default libeth settings * @fq: buffer queue struct to fill - * @napi: &napi_struct covering this PP (no usage outside its poll loops) + * @napi_dev: &napi_struct for NAPI (data) queues, &device for others * * Return: %0 on success, -%errno on failure. */ -int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi) +int libeth_rx_fq_create(struct libeth_fq *fq, void *napi_dev) { + struct napi_struct *napi = fq->no_napi ? NULL : napi_dev; struct page_pool_params pp = { .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, .order = LIBETH_RX_PAGE_ORDER, .pool_size = fq->count, .nid = fq->nid, - .dev = napi->dev->dev.parent, - .netdev = napi->dev, + .dev = napi ? napi->dev->dev.parent : napi_dev, + .netdev = napi ? napi->dev : NULL, .napi = napi, }; struct libeth_fqe *fqes; struct page_pool *pool; int ret; + if (!pp.netdev && fq->type == LIBETH_FQE_MTU) + return -EINVAL; + pp.dma_dir = fq->xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; if (!fq->hsplit) diff --git a/drivers/net/ethernet/intel/libie/Kconfig b/drivers/net/ethernet/intel/libie/Kconfig index 70831c7e336e..9c5fdebb6766 100644 --- a/drivers/net/ethernet/intel/libie/Kconfig +++ b/drivers/net/ethernet/intel/libie/Kconfig @@ -15,6 +15,14 @@ config LIBIE_ADMINQ Helper functions used by Intel Ethernet drivers for administration queue command interface (aka adminq). +config LIBIE_CP + tristate + select LIBETH + select LIBIE_PCI + help + Common helper routines to communicate with the device Control Plane + using virtchnl2 or related mailbox protocols. + config LIBIE_FWLOG tristate select LIBIE_ADMINQ @@ -23,3 +31,9 @@ config LIBIE_FWLOG for it. Firmware logging is using admin queue interface to communicate with the device. Debugfs is a user interface used to config logging and dump all collected logs. + +config LIBIE_PCI + tristate + help + Helper functions for management of PCI resources belonging + to networking devices. diff --git a/drivers/net/ethernet/intel/libie/Makefile b/drivers/net/ethernet/intel/libie/Makefile index db57fc6780ea..3065aa057798 100644 --- a/drivers/net/ethernet/intel/libie/Makefile +++ b/drivers/net/ethernet/intel/libie/Makefile @@ -9,6 +9,14 @@ obj-$(CONFIG_LIBIE_ADMINQ) += libie_adminq.o libie_adminq-y := adminq.o +obj-$(CONFIG_LIBIE_CP) += libie_cp.o + +libie_cp-y := controlq.o + obj-$(CONFIG_LIBIE_FWLOG) += libie_fwlog.o libie_fwlog-y := fwlog.o + +obj-$(CONFIG_LIBIE_PCI) += libie_pci.o + +libie_pci-y := pci.o diff --git a/drivers/net/ethernet/intel/libie/controlq.c b/drivers/net/ethernet/intel/libie/controlq.c new file mode 100644 index 000000000000..45a49eba6a82 --- /dev/null +++ b/drivers/net/ethernet/intel/libie/controlq.c @@ -0,0 +1,1295 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include +#include + +#include + +#define LIBIE_CTLQ_DESC_QWORD0(sz) \ + (LIBIE_CTLQ_DESC_FLAG_BUF | \ + LIBIE_CTLQ_DESC_FLAG_RD | \ + FIELD_PREP(LIBIE_CTLQ_DESC_DATA_LEN, sz)) + +/** + * libie_ctlq_free_fq - free fill queue resources, including buffers + * @ctlq: Rx control queue whose resources need to be freed + */ +static void libie_ctlq_free_fq(struct libie_ctlq_info *ctlq) +{ + struct libeth_fq fq = { + .fqes = ctlq->rx_fqes, + .pp = ctlq->pp, + }; + + for (u32 ntc = ctlq->next_to_clean; ntc != ctlq->next_to_post; ) { + page_pool_put_full_netmem(fq.pp, fq.fqes[ntc].netmem, false); + + if (++ntc >= ctlq->ring_len) + ntc = 0; + } + + libeth_rx_fq_destroy(&fq); +} + +/** + * libie_ctlq_init_fq - initialize fill queue for an Rx controlq + * @ctlq: control queue that needs Rx buffer allocation + * + * Return: %0 on success, -%errno on failure + */ +static int libie_ctlq_init_fq(struct libie_ctlq_info *ctlq) +{ + struct libeth_fq fq = { + .count = ctlq->ring_len, + .truesize = LIBIE_CTLQ_MAX_BUF_LEN, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_SHORT, + .hsplit = true, + .no_napi = true, + }; + int err; + + err = libeth_rx_fq_create(&fq, ctlq->dev); + if (err) + return err; + + ctlq->pp = fq.pp; + ctlq->rx_fqes = fq.fqes; + ctlq->truesize = fq.truesize; + + return 0; +} + +/** + * libie_ctlq_prep_rx_desc - prepare the descriptor with a new address + * @desc: descriptor to (re)initialize + * @addr: physical address to put into descriptor + * @mem_truesize: size of the accessible memory + */ +static void libie_ctlq_prep_rx_desc(struct libie_ctlq_desc *desc, + dma_addr_t addr, u32 mem_truesize) +{ + u64 qword; + + qword = LIBIE_CTLQ_DESC_QWORD0(mem_truesize); + desc->qword0 = cpu_to_le64(qword); + + qword = FIELD_PREP(LIBIE_CTLQ_DESC_DATA_ADDR_HIGH, + upper_32_bits(addr)) | + FIELD_PREP(LIBIE_CTLQ_DESC_DATA_ADDR_LOW, + lower_32_bits(addr)); + desc->qword3 = cpu_to_le64(qword); +} + +/** + * libie_ctlq_post_rx_buffs - post buffers to descriptor ring + * @ctlq: control queue that requires Rx descriptor ring to be initialized with + * new Rx buffers + * + * The caller must make sure that calls to libie_ctlq_post_rx_buffs() + * and libie_ctlq_recv() for each queue are either serialized + * or used under ctlq->lock. + * + * Return: %0 on success, -%ENOMEM if any buffer could not be allocated + */ +int libie_ctlq_post_rx_buffs(struct libie_ctlq_info *ctlq) +{ + u32 ntp = ctlq->next_to_post, ntc = ctlq->next_to_clean, num_to_post; + const struct libeth_fq_fp fq = { + .pp = ctlq->pp, + .fqes = ctlq->rx_fqes, + .truesize = ctlq->truesize, + .count = ctlq->ring_len, + }; + int ret = 0; + + num_to_post = (ntc > ntp ? 0 : ctlq->ring_len) + ntc - ntp - 1; + + while (num_to_post--) { + dma_addr_t addr; + + ctlq->descs[ntp] = (struct libie_ctlq_desc) {}; + + addr = libeth_rx_alloc(&fq, ntp); + if (unlikely(addr == DMA_MAPPING_ERROR)) { + ret = -ENOMEM; + goto post_bufs; + } + + libie_ctlq_prep_rx_desc(&ctlq->descs[ntp], addr, fq.truesize); + + if (unlikely(++ntp == ctlq->ring_len)) + ntp = 0; + } + +post_bufs: + if (likely(ctlq->next_to_post != ntp)) { + ctlq->next_to_post = ntp; + + dma_wmb(); + writel(ntp, ctlq->reg.tail); + } + + return ret; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_post_rx_buffs, "LIBIE_CP"); + +/** + * libie_ctlq_free_tx_msgs - Free Tx control queue messages + * @ctlq: Tx control queue being destroyed + * @num_msgs: number of messages allocated so far + */ +static void libie_ctlq_free_tx_msgs(struct libie_ctlq_info *ctlq, + u32 num_msgs) +{ + for (u32 i = 0; i < num_msgs; i++) + kfree(ctlq->tx_msg[i]); + + kvfree(ctlq->tx_msg); +} + +/** + * libie_ctlq_alloc_tx_msgs - Allocate Tx control queue messages + * @ctlq: Tx control queue being created + * + * Return: %0 on success, -%ENOMEM on allocation error + */ +static int libie_ctlq_alloc_tx_msgs(struct libie_ctlq_info *ctlq) +{ + ctlq->tx_msg = kvzalloc_objs(*ctlq->tx_msg, ctlq->ring_len, + GFP_KERNEL); + if (!ctlq->tx_msg) + return -ENOMEM; + + for (u32 i = 0; i < ctlq->ring_len; i++) { + ctlq->tx_msg[i] = kzalloc_obj(*ctlq->tx_msg[i]); + if (!ctlq->tx_msg[i]) { + libie_ctlq_free_tx_msgs(ctlq, i); + return -ENOMEM; + } + } + + return 0; +} + +/** + * libie_cp_free_desc_mem - free the previously allocated descriptor DMA memory + * @dev: device information + * @mem: DMA memory information + */ +static void libie_cp_free_desc_mem(struct device *dev, + struct libie_cp_dma_mem *mem) +{ + dma_free_coherent(dev, mem->size, mem->va, mem->pa); + mem->va = NULL; +} + +/** + * libie_ctlq_dealloc_ring_res - free memory allocated for control queue + * @ctlq: control queue that requires its ring memory to be freed + * + * Free the memory used by the ring, buffers and other related structures. + */ +static void libie_ctlq_dealloc_ring_res(struct libie_ctlq_info *ctlq) +{ + struct libie_cp_dma_mem *dma = &ctlq->ring_mem; + + if (ctlq->type == LIBIE_CTLQ_TYPE_TX) + libie_ctlq_free_tx_msgs(ctlq, ctlq->ring_len); + else + libie_ctlq_free_fq(ctlq); + + libie_cp_free_desc_mem(ctlq->dev, dma); +} + +/** + * libie_cp_alloc_desc_mem - allocate DMA memory for descriptor ring + * @dev: device information + * @mem: memory for DMA information to be stored + * @size: size of the memory to allocate + * + * Return: virtual address of DMA memory or NULL. + */ +static void *libie_cp_alloc_desc_mem(struct device *dev, + struct libie_cp_dma_mem *mem, u32 size) +{ + size = LARGEST_ALIGN(size); + + mem->va = dma_alloc_coherent(dev, size, &mem->pa, GFP_KERNEL); + mem->size = size; + mem->direction = DMA_BIDIRECTIONAL; + + return mem->va; +} + +/** + * libie_ctlq_alloc_queue_res - allocate memory for descriptor ring and bufs + * @ctlq: control queue that requires its ring resources to be allocated + * + * Return: %0 on success, -%errno on failure + */ +static int libie_ctlq_alloc_queue_res(struct libie_ctlq_info *ctlq) +{ + size_t size = array_size(ctlq->ring_len, sizeof(*ctlq->descs)); + struct libie_cp_dma_mem *dma = &ctlq->ring_mem; + int err = -ENOMEM; + + if (!libie_cp_alloc_desc_mem(ctlq->dev, dma, size)) + return -ENOMEM; + + ctlq->descs = dma->va; + + if (ctlq->type == LIBIE_CTLQ_TYPE_TX) { + if (libie_ctlq_alloc_tx_msgs(ctlq)) + goto free_dma_mem; + } else { + err = libie_ctlq_init_fq(ctlq); + if (err) + goto free_dma_mem; + + err = libie_ctlq_post_rx_buffs(ctlq); + if (err) { + libie_ctlq_free_fq(ctlq); + goto free_dma_mem; + } + } + + return 0; + +free_dma_mem: + libie_cp_free_desc_mem(ctlq->dev, dma); + + return err; +} + +/** + * libie_ctlq_init_regs - Initialize control queue registers + * @ctlq: control queue that needs to be initialized + * + * Initialize registers. The caller is expected to have already initialized the + * descriptor ring memory and buffer memory. + */ +static void libie_ctlq_init_regs(struct libie_ctlq_info *ctlq) +{ + u32 dword; + + if (ctlq->type == LIBIE_CTLQ_TYPE_RX) + writel(ctlq->ring_len - 1, ctlq->reg.tail); + else + writel(0, ctlq->reg.tail); + + writel(0, ctlq->reg.head); + writel(lower_32_bits(ctlq->ring_mem.pa), ctlq->reg.addr_low); + writel(upper_32_bits(ctlq->ring_mem.pa), ctlq->reg.addr_high); + + dword = FIELD_PREP(LIBIE_CTLQ_MBX_ATQ_LEN, ctlq->ring_len) | + ctlq->reg.len_ena_mask; + writel(dword, ctlq->reg.len); +} + +/** + * libie_find_ctlq - find the controlq for the given id and type + * @ctx: libie CP context information + * @type: type of controlq to find + * @id: controlq id to find + * + * Return: control queue info pointer on success, NULL on failure + */ +struct libie_ctlq_info *libie_find_ctlq(struct libie_ctlq_ctx *ctx, + enum libie_ctlq_type type, + int id) +{ + struct libie_ctlq_info *cq; + + guard(spinlock)(&ctx->ctlqs_lock); + + list_for_each_entry(cq, &ctx->ctlqs, list) + if (cq->qid == id && cq->type == type) + return cq; + + return NULL; +} +EXPORT_SYMBOL_NS_GPL(libie_find_ctlq, "LIBIE_CP"); + +/** + * libie_ctlq_add - add one control queue + * @ctx: libie CP context information + * @qinfo: information required for queue creation + * + * Allocate and initialize a control queue and add it to the control queue list. + * libie_ctlq_init() must be called prior to any calls to libie_ctlq_add. + * + * Return: added control queue info pointer on success, error pointer on failure + */ +static struct libie_ctlq_info * +libie_ctlq_add(struct libie_ctlq_ctx *ctx, + const struct libie_ctlq_create_info *qinfo) +{ + struct libie_ctlq_info *ctlq; + int err; + + if (qinfo->id != LIBIE_CTLQ_MBX_ID) + return ERR_PTR(-EOPNOTSUPP); + + if (qinfo->len > FIELD_MAX(LIBIE_CTLQ_MBX_ATQ_LEN) || !qinfo->len) + return ERR_PTR(-EINVAL); + + ctlq = kvzalloc_obj(*ctlq); + if (!ctlq) + return ERR_PTR(-ENOMEM); + + ctlq->type = qinfo->type; + ctlq->qid = qinfo->id; + ctlq->ring_len = qinfo->len; + ctlq->dev = &ctx->mmio_info.pdev->dev; + ctlq->reg = qinfo->reg; + + err = libie_ctlq_alloc_queue_res(ctlq); + if (err) { + kvfree(ctlq); + return ERR_PTR(err); + } + + libie_ctlq_init_regs(ctlq); + + spin_lock_init(&ctlq->lock); + + scoped_guard(spinlock, &ctx->ctlqs_lock) + list_add(&ctlq->list, &ctx->ctlqs); + + return ctlq; +} + +/** + * libie_ctlq_remove - deallocate and remove specified control queue + * @ctx: libie CP context information + * @ctlq: specific control queue that needs to be removed + */ +static void libie_ctlq_remove(struct libie_ctlq_ctx *ctx, + struct libie_ctlq_info *ctlq) +{ + scoped_guard(spinlock, &ctx->ctlqs_lock) + list_del(&ctlq->list); + + libie_ctlq_dealloc_ring_res(ctlq); + kvfree(ctlq); +} + +/** + * libie_ctlq_init - main initialization routine for all control queues + * @ctx: libie CP context information + * @qinfo: array of structs containing info for each queue to be initialized + * @numq: number of queues to initialize + * + * This initializes queue list and adds any number and any type of control + * queues. This is an all or nothing routine; if one fails, all previously + * allocated queues will be destroyed. + * + * Please note that any control queue send/receive functions are not + * softirq/NAPI safe, and therefore API can be used in process context only. + * + * Return: %0 on success, -%errno on failure + */ +int libie_ctlq_init(struct libie_ctlq_ctx *ctx, + const struct libie_ctlq_create_info *qinfo, + u32 numq) +{ + INIT_LIST_HEAD(&ctx->ctlqs); + spin_lock_init(&ctx->ctlqs_lock); + + for (u32 i = 0; i < numq; i++) { + struct libie_ctlq_info *ctlq; + + ctlq = libie_ctlq_add(ctx, &qinfo[i]); + if (IS_ERR(ctlq)) { + libie_ctlq_deinit(ctx); + return PTR_ERR(ctlq); + } + } + + return 0; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_init, "LIBIE_CP"); + +/** + * libie_ctlq_deinit - destroy all control queues + * @ctx: libie CP context information + */ +void libie_ctlq_deinit(struct libie_ctlq_ctx *ctx) +{ + struct libie_ctlq_info *ctlq, *tmp; + + list_for_each_entry_safe(ctlq, tmp, &ctx->ctlqs, list) + libie_ctlq_remove(ctx, ctlq); +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_deinit, "LIBIE_CP"); + +/** + * libie_ctlq_tx_desc_from_msg - initialize a Tx descriptor from a message + * @desc: descriptor to be initialized + * @msg: filled control queue message + */ +static void libie_ctlq_tx_desc_from_msg(struct libie_ctlq_desc *desc, + const struct libie_ctlq_msg *msg) +{ + const struct libie_cp_dma_mem *dma = &msg->send_mem; + u64 qword; + + qword = FIELD_PREP(LIBIE_CTLQ_DESC_FLAGS, msg->flags) | + FIELD_PREP(LIBIE_CTLQ_DESC_INFRA_OPCODE, msg->opcode) | + FIELD_PREP(LIBIE_CTLQ_DESC_PFID_VFID, msg->func_id); + desc->qword0 = cpu_to_le64(qword); + + qword = FIELD_PREP(LIBIE_CTLQ_DESC_VIRTCHNL_OPCODE, + msg->chnl_opcode) | + FIELD_PREP(LIBIE_CTLQ_DESC_VIRTCHNL_MSG_RET_VAL, + msg->chnl_retval); + desc->qword1 = cpu_to_le64(qword); + + qword = FIELD_PREP(LIBIE_CTLQ_DESC_MSG_PARAM0, msg->param0) | + FIELD_PREP(LIBIE_CTLQ_DESC_SW_COOKIE, + msg->sw_cookie) | + FIELD_PREP(LIBIE_CTLQ_DESC_VIRTCHNL_FLAGS, + msg->virt_flags); + desc->qword2 = cpu_to_le64(qword); + + if (likely(msg->data_len)) { + desc->qword0 |= + cpu_to_le64(LIBIE_CTLQ_DESC_QWORD0(msg->data_len)); + qword = FIELD_PREP(LIBIE_CTLQ_DESC_DATA_ADDR_HIGH, + upper_32_bits(dma->pa)) | + FIELD_PREP(LIBIE_CTLQ_DESC_DATA_ADDR_LOW, + lower_32_bits(dma->pa)); + } else { + qword = msg->addr_param; + } + + desc->qword3 = cpu_to_le64(qword); +} + +/** + * libie_ctlq_send_desc_avail - get number of free descriptors on a Tx ctlq + * @ctlq: specific control queue which is going be used for sending messages + * + * The caller must hold ctlq->lock. Any dependent sending must be done + * in the same critical section. + * + * Return: number of available descriptors/messages on a given control queue. + */ +u32 libie_ctlq_send_desc_avail(const struct libie_ctlq_info *ctlq) +{ + u32 ntu = ctlq->next_to_use, ntc = ctlq->next_to_clean; + + lockdep_assert_held(&ctlq->lock); + + return (ntc > ntu ? 0 : ctlq->ring_len) + ntc - ntu - 1; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_send_desc_avail, "LIBIE_CP"); + +/** + * libie_ctlq_send - send a message to Control Plane or Peer + * @ctlq: specific control queue which is used for sending a message + * @num_q_msg: number of messages present to send on @ctlq, + * positive and no greater than the number of available descriptors + * + * The caller must fill in @num_q_msg Tx messages starting at ntu beforehand. + * + * The caller must hold ctlq->lock. The intended pattern is to first check + * the number of descriptors available, then fill in the messages and perform + * send within a single critical section. + */ +void libie_ctlq_send(struct libie_ctlq_info *ctlq, u32 num_q_msg) +{ + u32 ntu = ctlq->next_to_use; + + lockdep_assert_held(&ctlq->lock); + + for (int i = 0; i < num_q_msg; i++) { + struct libie_ctlq_msg *msg = ctlq->tx_msg[ntu]; + struct libie_ctlq_desc *desc; + + desc = &ctlq->descs[ntu]; + libie_ctlq_tx_desc_from_msg(desc, msg); + + if (unlikely(++ntu == ctlq->ring_len)) + ntu = 0; + } + dma_wmb(); + writel(ntu, ctlq->reg.tail); + ctlq->next_to_use = ntu; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_send, "LIBIE_CP"); + +/** + * libie_ctlq_send_clean - cleanup the send control queue message buffers + * @params: information for handling of Tx completions + * + * Cleanup the send buffers for the given control queue, if force is set, then + * clear all the outstanding send messages irrespective of their send status, + * until a zero-length message is encountered, which is either a message that + * is already cleared, or a VF reset message, which is always last. + * Force should be used during deinit or reset. + * + * Return: number of send buffers cleaned. + */ +u32 libie_ctlq_send_clean(const struct libie_ctlq_clean_params *params) +{ + struct libie_ctlq_info *ctlq = params->ctlq; + u32 ntc, i; + + spin_lock(&ctlq->lock); + ntc = ctlq->next_to_clean; + + for (i = 0; i < params->num_msgs; i++) { + struct libie_ctlq_msg *msg = ctlq->tx_msg[ntc]; + struct libie_ctlq_desc *desc; + u64 qword; + + desc = &ctlq->descs[ntc]; + qword = le64_to_cpu(desc->qword0); + + if (!FIELD_GET(LIBIE_CTLQ_DESC_FLAG_DD, qword) && + !(unlikely(params->force) && msg->data_len)) + break; + + /* This cannot be reordered and lock is taken, so no barriers */ + desc->qword0 = 0; + + params->rel_dma_mem(params->rel_ctx, &msg->send_mem); + memset(msg, 0, sizeof(*msg)); + + if (unlikely(++ntc == ctlq->ring_len)) + ntc = 0; + } + + ctlq->next_to_clean = ntc; + spin_unlock(&ctlq->lock); + + return i; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_send_clean, "LIBIE_CP"); + +/** + * libie_ctlq_fill_rx_msg - fill in a message from Rx descriptor and buffer + * @msg: message to be filled in + * @desc: received descriptor + * @rx_buf: fill queue buffer associated with the descriptor + */ +static void libie_ctlq_fill_rx_msg(struct libie_ctlq_msg *msg, + const struct libie_ctlq_desc *desc, + struct libeth_fqe *rx_buf) +{ + u64 qword = le64_to_cpu(desc->qword0); + + msg->flags = FIELD_GET(LIBIE_CTLQ_DESC_FLAGS, qword); + msg->opcode = FIELD_GET(LIBIE_CTLQ_DESC_INFRA_OPCODE, qword); + msg->data_len = FIELD_GET(LIBIE_CTLQ_DESC_DATA_LEN, qword); + msg->hw_retval = FIELD_GET(LIBIE_CTLQ_DESC_HW_RETVAL, qword); + + qword = le64_to_cpu(desc->qword1); + msg->chnl_opcode = + FIELD_GET(LIBIE_CTLQ_DESC_VIRTCHNL_OPCODE, qword); + msg->chnl_retval = + FIELD_GET(LIBIE_CTLQ_DESC_VIRTCHNL_MSG_RET_VAL, qword); + + qword = le64_to_cpu(desc->qword2); + msg->param0 = + FIELD_GET(LIBIE_CTLQ_DESC_MSG_PARAM0, qword); + msg->sw_cookie = + FIELD_GET(LIBIE_CTLQ_DESC_SW_COOKIE, qword); + msg->virt_flags = + FIELD_GET(LIBIE_CTLQ_DESC_VIRTCHNL_FLAGS, qword); + + if (likely(msg->data_len)) { + if (unlikely(msg->data_len > LIBIE_CTLQ_MAX_BUF_LEN)) { + msg->data_len = LIBIE_CTLQ_MAX_BUF_LEN; + msg->chnl_retval = U32_MAX; + } + msg->recv_mem = (struct kvec) { + .iov_base = netmem_address(rx_buf->netmem) + + rx_buf->offset, + .iov_len = msg->data_len, + }; + libeth_rx_sync_for_cpu(rx_buf, msg->data_len); + } else { + msg->recv_mem = (struct kvec) {}; + msg->addr_param = le64_to_cpu(desc->qword3); + page_pool_put_full_netmem(netmem_get_pp(rx_buf->netmem), + rx_buf->netmem, false); + } +} + +/** + * libie_ctlq_recv - receive control queue messages + * @ctlq: control queue that needs to processed for receive + * @msg: array of received control queue messages on this q; + * needs to be pre-allocated by caller for as many messages as requested + * @num_q_msg: number of messages that can be stored in msg buffer, + * no greater than number of posted buffers + * + * Caller is expected to return buffers via libie_ctlq_release_rx_buf(). + * + * The caller must make sure that calls to libie_ctlq_post_rx_buffs() + * and libie_ctlq_recv() for each queue are either serialized + * or used under ctlq->lock. + * + * Return: number of messages received + */ +u32 libie_ctlq_recv(struct libie_ctlq_info *ctlq, struct libie_ctlq_msg *msg, + u32 num_q_msg) +{ + u32 ntc, i; + + ntc = ctlq->next_to_clean; + + for (i = 0; i < num_q_msg; i++) { + struct libie_ctlq_desc *desc = &ctlq->descs[ntc]; + struct libeth_fqe *rx_buf = &ctlq->rx_fqes[ntc]; + u64 qword; + + qword = le64_to_cpu(desc->qword0); + if (!FIELD_GET(LIBIE_CTLQ_DESC_FLAG_DD, qword)) + break; + + dma_rmb(); + + libie_ctlq_fill_rx_msg(&msg[i], desc, rx_buf); + desc->qword0 = 0; + + if (unlikely(++ntc == ctlq->ring_len)) + ntc = 0; + } + + ctlq->next_to_clean = ntc; + + return i; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_recv, "LIBIE_CP"); + +/** + * libie_ctlq_xn_pop_free - get a free Xn entry from the free list + * @xnm: Xn transaction manager + * + * Retrieve a free Xn entry from the free list. + * + * Return: valid Xn entry pointer or NULL if there are no free Xn entries. + */ +static struct libie_ctlq_xn * +libie_ctlq_xn_pop_free(struct libie_ctlq_xn_manager *xnm) +{ + struct libie_ctlq_xn *xn; + u32 free_idx; + + guard(spinlock)(&xnm->free_xns_bm_lock); + + if (unlikely(xnm->shutdown)) + return NULL; + + for_each_set_bit(free_idx, xnm->free_xns_bm, + LIBIE_CTLQ_MAX_XN_ENTRIES) { + xn = &xnm->ring[free_idx]; + + /* Torn read of the physical address is possible, the worst case + * scenario is a transient spurious skip. If the physical + * address is dirty in any way, reuse is already safe. + */ + if (xn->tx_msg && + data_race(xn->tx_msg->send_mem.pa) == xn->small_dma_mem.pa) + continue; + + clear_bit(free_idx, xnm->free_xns_bm); + + return xn; + } + + return NULL; +} + +/** + * __libie_ctlq_xn_push_free - unsafely push an xn entry into the free list + * @xnm: Xn transaction manager + * @xn: xn entry to be added into the free list + * + * Return: whether xnm destruction can be triggered by the caller + */ +static bool __libie_ctlq_xn_push_free(struct libie_ctlq_xn_manager *xnm, + struct libie_ctlq_xn *xn) +{ + xn->cookie++; + set_bit(xn->index, xnm->free_xns_bm); + + if (unlikely(xnm->shutdown) && + bitmap_full(xnm->free_xns_bm, LIBIE_CTLQ_MAX_XN_ENTRIES)) + return true; + + return false; +} + +/** + * libie_ctlq_xn_push_free - push a Xn entry into the free list + * @xnm: Xn transaction manager + * @xn: xn entry to be added into the free list, not locked + * + * Safely add a used Xn entry back to the free list. + */ +static void libie_ctlq_xn_push_free(struct libie_ctlq_xn_manager *xnm, + struct libie_ctlq_xn *xn) +{ + bool can_destroy; + + scoped_guard(spinlock, &xnm->free_xns_bm_lock) + can_destroy = __libie_ctlq_xn_push_free(xnm, xn); + + if (can_destroy) + complete(&xnm->can_destroy); +} + +/** + * libie_ctlq_xn_deinit_dma - free the DMA memory allocated for send messages + * @xnm: pointer to the transaction manager + * @num_entries: number of Xn entries to free the DMA for + */ +static void libie_ctlq_xn_deinit_dma(struct libie_ctlq_xn_manager *xnm, + u32 num_entries) +{ + for (u32 i = 0; i < num_entries; i++) { + struct libie_ctlq_xn *xn = &xnm->ring[i]; + + dma_pool_free(xnm->small_buff_pool, xn->small_dma_mem.va, + xn->small_dma_mem.pa); + } + + dma_pool_destroy(xnm->small_buff_pool); +} + +/** + * libie_ctlq_xn_init_dma - pre-allocate DMA memory for send messages that use + * stack variables + * @dev: device pointer + * @xnm: pointer to transaction manager + * + * Return: %0 on success or error if memory allocation fails + */ +static int libie_ctlq_xn_init_dma(struct device *dev, + struct libie_ctlq_xn_manager *xnm) +{ + u32 i; + + xnm->small_buff_pool = + dma_pool_create("libie_ctlq_xn_tx", dev, LIBIE_CP_TX_COPYBREAK, + LIBIE_CP_TX_COPYBREAK, 0); + if (!xnm->small_buff_pool) + return -ENOMEM; + + for (i = 0; i < LIBIE_CTLQ_MAX_XN_ENTRIES; i++) { + struct libie_cp_dma_mem *mem = &xnm->ring[i].small_dma_mem; + + mem->va = dma_pool_zalloc(xnm->small_buff_pool, GFP_KERNEL, + &mem->pa); + if (!mem->va) + goto dealloc_dma; + + mem->direction = DMA_BIDIRECTIONAL; + mem->size = LIBIE_CP_TX_COPYBREAK; + } + + return 0; + +dealloc_dma: + libie_ctlq_xn_deinit_dma(xnm, i); + + return -ENOMEM; +} + +/** + * libie_ctlq_xn_process_recv - process Xn data in receive message + * @params: Xn receive param information to handle a receive message + * @ctlq_msg: received control queue message + * + * Process a control queue receive message and send a complete event + * notification. + * + * Return: true if a message has been processed, false otherwise. + */ +static bool +libie_ctlq_xn_process_recv(struct libie_ctlq_xn_recv_params *params, + struct libie_ctlq_msg *ctlq_msg) +{ + struct libie_ctlq_xn_manager *xnm = params->xnm; + struct libie_ctlq_xn *xn; + u16 msg_cookie, xn_index; + struct kvec *response; + int status; + u16 data; + + data = ctlq_msg->sw_cookie; + xn_index = FIELD_GET(LIBIE_CTLQ_XN_INDEX_M, data); + msg_cookie = FIELD_GET(LIBIE_CTLQ_XN_COOKIE_M, data); + status = ctlq_msg->chnl_retval ? -EFAULT : 0; + + xn = &xnm->ring[xn_index]; + spin_lock(&xn->xn_lock); + if (ctlq_msg->chnl_opcode != xn->virtchnl_opcode || + msg_cookie != xn->cookie) { + spin_unlock(&xn->xn_lock); + return false; + } + + if (xn->state != LIBIE_CTLQ_XN_ASYNC && + xn->state != LIBIE_CTLQ_XN_WAITING) { + spin_unlock(&xn->xn_lock); + return false; + } + + response = &ctlq_msg->recv_mem; + if (xn->state == LIBIE_CTLQ_XN_ASYNC) { + xn->resp_cb(xn->send_ctx, response, status); + libie_ctlq_release_rx_buf(response); + xn->state = LIBIE_CTLQ_XN_IDLE; + spin_unlock(&xn->xn_lock); + libie_ctlq_xn_push_free(xnm, xn); + + return true; + } + + xn->recv_mem = *response; + xn->state = status ? LIBIE_CTLQ_XN_COMPLETED_FAILED : + LIBIE_CTLQ_XN_COMPLETED_SUCCESS; + + complete(&xn->cmd_completion_event); + spin_unlock(&xn->xn_lock); + + return true; +} + +/** + * libie_xn_check_async_timeout - Check for asynchronous message timeouts + * @xnm: Xn transaction manager + * + * Call the corresponding callback to notify the caller about the timeout. + * Iterates free_xns_bm locklessly, potential races are caught under + * xn->xn_lock. + */ +static void libie_xn_check_async_timeout(struct libie_ctlq_xn_manager *xnm) +{ + u32 idx; + + for_each_clear_bit(idx, xnm->free_xns_bm, LIBIE_CTLQ_MAX_XN_ENTRIES) { + struct libie_ctlq_xn *xn = &xnm->ring[idx]; + u64 timeout_ms; + + spin_lock(&xn->xn_lock); + + timeout_ms = ktime_ms_delta(ktime_get(), xn->timestamp); + if (xn->state != LIBIE_CTLQ_XN_ASYNC || + timeout_ms < xn->timeout_ms) { + spin_unlock(&xn->xn_lock); + continue; + } + + xn->resp_cb(xn->send_ctx, NULL, -ETIMEDOUT); + xn->state = LIBIE_CTLQ_XN_IDLE; + spin_unlock(&xn->xn_lock); + libie_ctlq_xn_push_free(xnm, xn); + } +} + +/** + * libie_ctlq_xn_recv - process control queue receive message + * @params: Xn receive param information to handle a receive message + * + * Process a receive message and update the receive queue buffer. + * Also terminates async transactions for which it failed to receive a response + * within a given timeframe. + * Function is intended to be called periodically from a single task. + * + * Return: remaining budget. + */ +u32 libie_ctlq_xn_recv(struct libie_ctlq_xn_recv_params *params) +{ + struct libie_ctlq_msg ctlq_msg; + u32 budget = params->budget; + + while (budget && libie_ctlq_recv(params->ctlq, &ctlq_msg, 1)) { + budget--; + if (!libie_ctlq_xn_process_recv(params, &ctlq_msg)) + params->ctlq_msg_handler(params->xnm->ctx, &ctlq_msg); + } + + libie_ctlq_post_rx_buffs(params->ctlq); + libie_xn_check_async_timeout(params->xnm); + + return budget; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_xn_recv, "LIBIE_CP"); + +/** + * libie_cp_map_dma_mem - map a given virtual address for DMA + * @dev: device information + * @va: virtual address to be mapped + * @size: size of the memory + * @direction: DMA direction either from/to device + * @dma_mem: memory for DMA information to be stored + * + * Return: true on success, false on DMA map failure. + */ +static bool libie_cp_map_dma_mem(struct device *dev, void *va, size_t size, + int direction, + struct libie_cp_dma_mem *dma_mem) +{ + dma_mem->pa = dma_map_single(dev, va, size, direction); + + return dma_mapping_error(dev, dma_mem->pa) ? false : true; +} + +/** + * libie_cp_unmap_dma_mem - unmap previously mapped DMA address + * @dev: device information + * @dma_mem: DMA memory information + */ +static void libie_cp_unmap_dma_mem(struct device *dev, + const struct libie_cp_dma_mem *dma_mem) +{ + dma_unmap_single(dev, dma_mem->pa, dma_mem->size, + dma_mem->direction); +} + +/** + * libie_ctlq_xn_process_send - process and send a control queue message + * @params: Xn send param information for sending a control queue message + * @xn: Assigned Xn entry for tracking the control queue message + * + * Return: %0 on success, -%errno on failure. + */ +static +int libie_ctlq_xn_process_send(struct libie_ctlq_xn_send_params *params, + struct libie_ctlq_xn *xn) +{ + size_t buf_len = params->send_buf.iov_len; + struct device *dev = params->ctlq->dev; + void *buf = params->send_buf.iov_base; + struct libie_cp_dma_mem *dma_mem; + u16 cookie; + + if (!buf || !buf_len) + return -EOPNOTSUPP; + + if (libie_cp_can_send_onstack(buf_len)) { + dma_mem = &xn->small_dma_mem; + memcpy(dma_mem->va, buf, buf_len); + } else { + dma_mem = &xn->send_dma_mem; + dma_mem->va = buf; + dma_mem->size = buf_len; + dma_mem->direction = DMA_TO_DEVICE; + + if (!libie_cp_map_dma_mem(dev, buf, buf_len, DMA_TO_DEVICE, + dma_mem)) + return -ENOMEM; + } + + cookie = FIELD_PREP(LIBIE_CTLQ_XN_COOKIE_M, xn->cookie) | + FIELD_PREP(LIBIE_CTLQ_XN_INDEX_M, xn->index); + + scoped_guard(spinlock, ¶ms->ctlq->lock) { + struct libie_ctlq_info *ctlq = params->ctlq; + struct libie_ctlq_msg *ctlq_msg; + + if (!libie_ctlq_send_desc_avail(ctlq)) { + if (!libie_cp_can_send_onstack(buf_len)) + libie_cp_unmap_dma_mem(dev, dma_mem); + + return -EBUSY; + } + + ctlq_msg = ctlq->tx_msg[ctlq->next_to_use]; + xn->tx_msg = dma_mem == &xn->small_dma_mem ? ctlq_msg : NULL; + if (params->ctlq_msg) + *ctlq_msg = *params->ctlq_msg; + else + /* Unused ctlq messages are already zeroed */ + ctlq_msg->opcode = LIBIE_CTLQ_SEND_MSG_TO_CP; + + ctlq_msg->sw_cookie = cookie; + ctlq_msg->send_mem = *dma_mem; + ctlq_msg->data_len = buf_len; + ctlq_msg->chnl_opcode = params->chnl_opcode; + libie_ctlq_send(params->ctlq, 1); + } + + return 0; +} + +/** + * libie_ctlq_xn_send - send a control queue message, initiating a transaction + * @params: Xn send param information for sending a control queue message + * + * Send a control queue (mailbox or config) message. + * Based on the params value, the call can be completed synchronously or + * asynchronously. + * + * Return: %0 on success, -%errno on failure. + */ +int libie_ctlq_xn_send(struct libie_ctlq_xn_send_params *params) +{ + bool free_send = !libie_cp_can_send_onstack(params->send_buf.iov_len); + struct libie_ctlq_xn *xn; + int ret; + + if (params->send_buf.iov_len > LIBIE_CTLQ_MAX_BUF_LEN) { + ret = -EINVAL; + goto free_buf; + } + + xn = libie_ctlq_xn_pop_free(params->xnm); + /* no free transactions available */ + if (unlikely(!xn)) { + ret = -EAGAIN; + goto free_buf; + } + + spin_lock(&xn->xn_lock); + if (xn->state == LIBIE_CTLQ_XN_SHUTDOWN) { + ret = -ENXIO; + goto unlock_xn; + } + + xn->state = params->resp_cb ? LIBIE_CTLQ_XN_ASYNC : + LIBIE_CTLQ_XN_WAITING; + xn->virtchnl_opcode = params->chnl_opcode; + + if (params->resp_cb) { + xn->send_ctx = params->send_ctx; + xn->resp_cb = params->resp_cb; + xn->timeout_ms = params->timeout_ms; + xn->timestamp = ktime_get(); + } + + ret = libie_ctlq_xn_process_send(params, xn); + if (ret) + goto release_xn; + else + free_send = false; + + spin_unlock(&xn->xn_lock); + + if (params->resp_cb) + return 0; + + wait_for_completion_timeout(&xn->cmd_completion_event, + msecs_to_jiffies(params->timeout_ms)); + + spin_lock(&xn->xn_lock); + switch (xn->state) { + case LIBIE_CTLQ_XN_WAITING: + ret = -ETIMEDOUT; + break; + case LIBIE_CTLQ_XN_COMPLETED_SUCCESS: + params->recv_mem = xn->recv_mem; + break; + default: + ret = -EBADMSG; + break; + } + + /* Free the receive buffer in case of failure. On timeout, receive + * buffer is not allocated. + */ + if (ret && ret != -ETIMEDOUT) + libie_ctlq_release_rx_buf(&xn->recv_mem); + +release_xn: + xn->state = LIBIE_CTLQ_XN_IDLE; + reinit_completion(&xn->cmd_completion_event); +unlock_xn: + spin_unlock(&xn->xn_lock); + libie_ctlq_xn_push_free(params->xnm, xn); +free_buf: + if (free_send) + params->rel_tx_buf(params->send_buf.iov_base); + + return ret; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_xn_send, "LIBIE_CP"); + +/** + * struct libie_ctlq_xn_rel_tx_ctx - context needed to release xn Tx message + * @dev: device for which DMA was mapped + * @rel_tx_buf: freeing function for non-small buffers + */ +struct libie_ctlq_xn_rel_tx_ctx { + struct device *dev; + void (*rel_tx_buf)(const void *buf_va); +}; + +/** + * libie_ctlq_xn_rel_tx_buf - release xn-controlled Tx message buffer + * @ctx: context, namely DMA device and freeing function + * @dma_mem: DMA memory to reclaim/unmap + */ +static void libie_ctlq_xn_rel_tx_buf(const void *ctx, + struct libie_cp_dma_mem *dma_mem) +{ + const struct libie_ctlq_xn_rel_tx_ctx *rel_ctx = ctx; + + if (!libie_cp_can_send_onstack(dma_mem->size)) { + libie_cp_unmap_dma_mem(rel_ctx->dev, dma_mem); + rel_ctx->rel_tx_buf(dma_mem->va); + } +} + +/** + * libie_ctlq_xn_send_clean - clean xn-controlled Tx messages + * @ctlq: control queue to clean + * @rel_tx_buf: driver callback to free the buffer + * @force: clean regardless of DD + * + * Return: number of completed/released messages. + */ +u32 libie_ctlq_xn_send_clean(struct libie_ctlq_info *ctlq, + void (*rel_tx_buf)(const void *buf_va), + bool force) +{ + struct libie_ctlq_xn_rel_tx_ctx rel_ctx = { + .dev = ctlq->dev, + .rel_tx_buf = rel_tx_buf, + }; + struct libie_ctlq_clean_params params = { + .ctlq = ctlq, + .force = force, + .num_msgs = ctlq->ring_len, + .rel_ctx = &rel_ctx, + .rel_dma_mem = libie_ctlq_xn_rel_tx_buf, + }; + + return libie_ctlq_send_clean(¶ms); +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_xn_send_clean, "LIBIE_CP"); + +/** + * libie_ctlq_xn_shutdown - terminate control queue transactions + * @xnm: pointer to the transaction manager + * + * Synchronously terminate existing transactions and stop accepting new ones. + * Async transactions are discarded without invoking resp_cb. + */ +void libie_ctlq_xn_shutdown(struct libie_ctlq_xn_manager *xnm) +{ + bool must_wait = false; + u32 i; + + /* Should be no new clear bits after this */ + spin_lock(&xnm->free_xns_bm_lock); + xnm->shutdown = true; + + for_each_clear_bit(i, xnm->free_xns_bm, LIBIE_CTLQ_MAX_XN_ENTRIES) { + struct libie_ctlq_xn *xn = &xnm->ring[i]; + + spin_lock(&xn->xn_lock); + + switch (xn->state) { + /* if an idle xn is not free, it is about to be either + * freed or initialized, prevent the latter and wait + */ + case LIBIE_CTLQ_XN_IDLE: + xn->state = LIBIE_CTLQ_XN_SHUTDOWN; + fallthrough; + /* waiting thread possibly needs a push to return the xn, + * transaction will be reported as timed out + */ + case LIBIE_CTLQ_XN_WAITING: + complete(&xn->cmd_completion_event); + fallthrough; + /* these states will return the xn soon */ + case LIBIE_CTLQ_XN_COMPLETED_SUCCESS: + case LIBIE_CTLQ_XN_COMPLETED_FAILED: + case LIBIE_CTLQ_XN_SHUTDOWN: + must_wait = true; + break; + /* no thread should reference async xns at this point */ + case LIBIE_CTLQ_XN_ASYNC: + xn->state = LIBIE_CTLQ_XN_IDLE; + __libie_ctlq_xn_push_free(xnm, xn); + break; + } + + spin_unlock(&xn->xn_lock); + } + + spin_unlock(&xnm->free_xns_bm_lock); + + if (must_wait) + wait_for_completion(&xnm->can_destroy); +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_xn_shutdown, "LIBIE_CP"); + +/** + * libie_ctlq_xn_deinit - deallocate and free the transaction manager resources + * @xnm: pointer to the transaction manager + * @ctx: libie CP context information + * + * Rx processing must be stopped beforehand via cancelling tasks. + * Tx processing must be stopped beforehand via libie_ctlq_xn_shutdown(), + * all buffers must be force-cleaned from the send queue. + */ +void libie_ctlq_xn_deinit(struct libie_ctlq_xn_manager *xnm, + struct libie_ctlq_ctx *ctx) +{ + libie_ctlq_xn_deinit_dma(xnm, LIBIE_CTLQ_MAX_XN_ENTRIES); + kvfree(xnm); + libie_ctlq_deinit(ctx); +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_xn_deinit, "LIBIE_CP"); + +/** + * libie_ctlq_xn_init - initialize the Xn transaction manager + * @params: Xn init param information for allocating Xn manager resources + * + * Return: %0 on success, -%errno on failure. + */ +int libie_ctlq_xn_init(struct libie_ctlq_xn_init_params *params) +{ + struct libie_ctlq_xn_manager *xnm; + int ret; + + ret = libie_ctlq_init(params->ctx, params->cctlq_info, params->num_qs); + if (ret) + return ret; + + xnm = kvzalloc_obj(*xnm); + if (!xnm) + goto ctlq_deinit; + + ret = libie_ctlq_xn_init_dma(¶ms->ctx->mmio_info.pdev->dev, xnm); + if (ret) + goto free_xnm; + + spin_lock_init(&xnm->free_xns_bm_lock); + init_completion(&xnm->can_destroy); + bitmap_fill(xnm->free_xns_bm, LIBIE_CTLQ_MAX_XN_ENTRIES); + + for (u32 i = 0; i < LIBIE_CTLQ_MAX_XN_ENTRIES; i++) { + struct libie_ctlq_xn *xn = &xnm->ring[i]; + + xn->index = i; + init_completion(&xn->cmd_completion_event); + spin_lock_init(&xn->xn_lock); + } + xnm->ctx = params->ctx; + params->xnm = xnm; + + return 0; + +free_xnm: + kvfree(xnm); +ctlq_deinit: + libie_ctlq_deinit(params->ctx); + + return -ENOMEM; +} +EXPORT_SYMBOL_NS_GPL(libie_ctlq_xn_init, "LIBIE_CP"); + +MODULE_DESCRIPTION("Control Plane communication API"); +MODULE_IMPORT_NS("LIBETH"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/intel/libie/pci.c b/drivers/net/ethernet/intel/libie/pci.c new file mode 100644 index 000000000000..b756c1186ce9 --- /dev/null +++ b/drivers/net/ethernet/intel/libie/pci.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include + +/** + * libie_find_mmio_region - find MMIO region containing a range + * @mmio_list: list that contains MMIO region info + * @offset: range start offset + * @size: range size + * @bar_idx: BAR index containing the range to search + * + * MMIO regions mappings are traversed from oldest to newest. + * + * Return: pointer to a MMIO region overlapping with the range in any way or + * NULL if no such region is mapped. + */ +static struct libie_pci_mmio_region * +libie_find_mmio_region(const struct list_head *mmio_list, + resource_size_t offset, resource_size_t size, + int bar_idx) +{ + resource_size_t end_offset = offset + size; + struct libie_pci_mmio_region *mr; + + list_for_each_entry(mr, mmio_list, list) { + resource_size_t mr_end = mr->offset + mr->size; + resource_size_t mr_start = mr->offset; + + if (mr->bar_idx != bar_idx) + continue; + if (offset < mr_end && end_offset > mr_start) + return mr; + } + + return NULL; +} + +/** + * __libie_pci_get_mmio_addr - get the MMIO virtual address + * @mmio_info: contains list of MMIO regions + * @offset: register offset to find + * @num_args: number of additional arguments present + * @...: optional BAR index (0 by default) + * + * This function finds the virtual address of a register offset by iterating + * through the non-linear MMIO regions that are mapped by the driver. + * + * The list is traversed oldest to newest, so accessing an older mapping + * via this function while deleting a newer one is allowed. + * + * Return: valid MMIO virtual address or NULL. + */ +void __iomem *__libie_pci_get_mmio_addr(struct libie_mmio_info *mmio_info, + resource_size_t offset, + int num_args, ...) +{ + struct libie_pci_mmio_region *mr; + int bar_idx = 0; + va_list args; + + if (num_args) { + va_start(args, num_args); + bar_idx = va_arg(args, int); + va_end(args); + } + + list_for_each_entry(mr, &mmio_info->mmio_list, list) + if (bar_idx == mr->bar_idx && offset >= mr->offset && + offset < mr->offset + mr->size) { + offset -= mr->offset; + + return mr->addr + offset; + } + + WARN_ONCE(true, "Access to an unmapped MMIO region (BAR%d, offset %pa)", + bar_idx, &offset); + + return NULL; +} +EXPORT_SYMBOL_NS_GPL(__libie_pci_get_mmio_addr, "LIBIE_PCI"); + +/** + * __libie_pci_map_mmio_region - map PCI device MMIO region + * @mmio_info: struct to store the mapped MMIO region + * @offset: MMIO region start offset + * @size: MMIO region size + * @num_args: number of additional arguments present + * @...: optional BAR index (0 by default) + * + * Return: true if the requested address range is accessible through + * new or existing mapping, false otherwise. + */ +bool __libie_pci_map_mmio_region(struct libie_mmio_info *mmio_info, + resource_size_t offset, + resource_size_t size, int num_args, ...) +{ + struct pci_dev *pdev = mmio_info->pdev; + struct libie_pci_mmio_region *mr; + resource_size_t end_offset; + void __iomem *va; + int bar_idx = 0; + va_list args; + + if (num_args) { + va_start(args, num_args); + bar_idx = va_arg(args, int); + va_end(args); + } + + if (bar_idx >= PCI_STD_NUM_BARS || bar_idx < 0 || + !pci_resource_is_mem(pdev, bar_idx)) + return false; + + /* pci_iomap_range() silently maps less + * if the requested length is too big + */ + if (!size || check_add_overflow(offset, size, &end_offset) || + end_offset > pci_resource_len(pdev, bar_idx)) + return false; + + mr = libie_find_mmio_region(&mmio_info->mmio_list, offset, size, + bar_idx); + if (mr) { + pci_warn(pdev, + "Mapping of BAR%u (offset=%llu, size=%llu) intersecting region (offset=%llu, size=%llu) already exists\n", + bar_idx, (unsigned long long)mr->offset, + (unsigned long long)mr->size, + (unsigned long long)offset, (unsigned long long)size); + return mr->offset <= offset && + mr->offset + mr->size >= end_offset; + } + + va = pci_iomap_range(mmio_info->pdev, bar_idx, offset, size); + if (!va) { + pci_err(pdev, "Failed to map BAR%u region\n", bar_idx); + return false; + } + + mr = kvzalloc_obj(*mr); + if (!mr) { + pci_iounmap(pdev, va); + return false; + } + + mr->addr = va; + mr->offset = offset; + mr->size = size; + mr->bar_idx = bar_idx; + + list_add_tail(&mr->list, &mmio_info->mmio_list); + + return true; +} +EXPORT_SYMBOL_NS_GPL(__libie_pci_map_mmio_region, "LIBIE_PCI"); + +/** + * libie_pci_unmap_fltr_regs - unmap selected PCI device MMIO regions + * @mmio_info: contains list of MMIO regions to unmap + * @fltr: returns true, if region is to be unmapped + */ +void libie_pci_unmap_fltr_regs(struct libie_mmio_info *mmio_info, + bool (*fltr)(struct libie_mmio_info *mmio_info, + struct libie_pci_mmio_region *reg)) +{ + struct libie_pci_mmio_region *mr, *tmp; + + list_for_each_entry_safe(mr, tmp, &mmio_info->mmio_list, list) { + if (!fltr(mmio_info, mr)) + continue; + list_del(&mr->list); + pci_iounmap(mmio_info->pdev, mr->addr); + kvfree(mr); + } +} +EXPORT_SYMBOL_NS_GPL(libie_pci_unmap_fltr_regs, "LIBIE_PCI"); + +/** + * libie_pci_unmap_all_mmio_regions - unmap all PCI device MMIO regions + * @mmio_info: contains list of MMIO regions to unmap + */ +void libie_pci_unmap_all_mmio_regions(struct libie_mmio_info *mmio_info) +{ + struct libie_pci_mmio_region *mr, *tmp; + + list_for_each_entry_safe(mr, tmp, &mmio_info->mmio_list, list) { + list_del(&mr->list); + pci_iounmap(mmio_info->pdev, mr->addr); + kvfree(mr); + } +} +EXPORT_SYMBOL_NS_GPL(libie_pci_unmap_all_mmio_regions, "LIBIE_PCI"); + +/** + * libie_pci_init_dev - enable and configure the device + * @pdev: PCI device information + * + * Enable the device, request memory regions, set 64-bit DMA mask + * and coherent DMA mask, and enable bus-mastering + * + * Return: %0 on success, -%errno on failure. + */ +int libie_pci_init_dev(struct pci_dev *pdev) +{ + int err; + + err = pcim_enable_device(pdev); + if (err) + return err; + + for (int bar = 0; bar < PCI_STD_NUM_BARS; bar++) + if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) { + err = pcim_request_region(pdev, bar, pci_name(pdev)); + if (err) + return err; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) + return err; + + pci_set_master(pdev); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(libie_pci_init_dev, "LIBIE_PCI"); + +MODULE_DESCRIPTION("Common Ethernet PCI library"); +MODULE_LICENSE("GPL"); diff --git a/include/linux/net/intel/libie/controlq.h b/include/linux/net/intel/libie/controlq.h new file mode 100644 index 000000000000..175227d4eef1 --- /dev/null +++ b/include/linux/net/intel/libie/controlq.h @@ -0,0 +1,436 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef __LIBIE_CONTROLQ_H +#define __LIBIE_CONTROLQ_H + +#include +#include + +#include +#include + +/* Default mailbox control queue */ +#define LIBIE_CTLQ_MBX_ID -1 +#define LIBIE_CTLQ_MAX_BUF_LEN SZ_4K + +/** + * enum libie_ctlq_type - control queue type + * @LIBIE_CTLQ_TYPE_TX: basic Tx control queue + * @LIBIE_CTLQ_TYPE_RX: basic Rx control queue + */ +enum libie_ctlq_type { + LIBIE_CTLQ_TYPE_TX = 0, + LIBIE_CTLQ_TYPE_RX = 1, +}; + +/* Opcode used to send controlq message to the control plane */ +#define LIBIE_CTLQ_SEND_MSG_TO_CP 0x801 +#define LIBIE_CTLQ_SEND_MSG_TO_PEER 0x804 + +#define LIBIE_CP_TX_COPYBREAK 128 + +/** + * struct libie_ctlq_ctx - contains controlq info and MMIO region info + * @mmio_info: MMIO region info structure + * @ctlqs: list that stores all the control queues + * @ctlqs_lock: lock for control queue list + */ +struct libie_ctlq_ctx { + struct libie_mmio_info mmio_info; + struct list_head ctlqs; + spinlock_t ctlqs_lock; /* protects the ctlqs list */ +}; + +/** + * struct libie_ctlq_reg - structure representing virtual addresses of the + * controlq registers and masks + * @head: controlq head register address + * @tail: controlq tail register address + * @len: register address to write controlq length and enable bit + * @addr_high: register address to write the upper 32b of ring physical address + * @addr_low: register address to write the lower 32b of ring physical address + * @len_mask: mask to read the controlq length + * @len_ena_mask: mask to write the controlq enable bit + * @head_mask: mask to read the head value + */ +struct libie_ctlq_reg { + void __iomem *head; + void __iomem *tail; + void __iomem *len; + void __iomem *addr_high; + void __iomem *addr_low; + u32 len_mask; + u32 len_ena_mask; + u32 head_mask; +}; + +/** + * struct libie_cp_dma_mem - structure for DMA memory + * @va: virtual address + * @pa: physical address + * @size: memory size + * @direction: memory to device or device to memory + */ +struct libie_cp_dma_mem { + void *va; + dma_addr_t pa; + size_t size; + int direction; +}; + +/** + * struct libie_ctlq_msg - control queue message data + * @flags: refer to 'Flags sub-structure' definitions + * @opcode: infrastructure message opcode + * @data_len: size of the payload + * @func_id: queue id for mailbox selection, 0 for default mailbox (Tx) + * @hw_retval: execution status from the HW (Rx) + * @chnl_opcode: virtchnl message opcode + * @chnl_retval: virtchnl return value + * @param0: indirect message raw parameter0 + * @sw_cookie: used to verify the response of the sent virtchnl message + * @virt_flags: virtchnl capability flags + * @addr_param: additional parameters in place of the address, given no buffer + * @recv_mem: virtual address and size of the buffer that contains + * the indirect response + * @send_mem: physical and virtual address of the DMA buffer, + * used for sending + */ +struct libie_ctlq_msg { + u16 flags; + u16 opcode; + u16 data_len; + union { + u16 func_id; + u16 hw_retval; + }; + u32 chnl_opcode; + u32 chnl_retval; + u32 param0; + u16 sw_cookie; + u16 virt_flags; + u64 addr_param; + union { + struct kvec recv_mem; + struct libie_cp_dma_mem send_mem; + }; +}; + +/** + * struct libie_ctlq_create_info - control queue create information + * @type: control queue type (Rx or Tx) + * @id: queue offset passed as input, -1 for default mailbox + * @reg: registers accessed by control queue + * @len: controlq length + */ +struct libie_ctlq_create_info { + enum libie_ctlq_type type; + int id; + struct libie_ctlq_reg reg; + u16 len; +}; + +/** + * struct libie_ctlq_info - control queue information + * @list: used to add a controlq to the list of queues in libie_ctlq_ctx + * @type: control queue type + * @qid: queue identifier + * @lock: control queue lock + * @ring_mem: descriptor ring DMA memory + * @descs: array of descriptors + * @rx_fqes: array of controlq Rx buffers + * @tx_msg: Tx messages sent to hardware + * @reg: registers used by control queue + * @dev: device that owns this control queue + * @pp: page pool for controlq Rx buffers + * @truesize: size to allocate per buffer + * @next_to_clean: next descriptor to be cleaned + * @next_to_use: next available slot to send buffer (Tx queue) + * @next_to_post: next available slot to post buffers to (Rx queue) + * @ring_len: length of the descriptor ring + */ +struct libie_ctlq_info { + struct list_head list; + enum libie_ctlq_type type; + int qid; + spinlock_t lock; /* for concurrent processing */ + struct libie_cp_dma_mem ring_mem; + struct libie_ctlq_desc *descs; + union { + struct libeth_fqe *rx_fqes; + struct libie_ctlq_msg **tx_msg; + }; + struct libie_ctlq_reg reg; + struct device *dev; + struct page_pool *pp; + u32 truesize; + u32 next_to_clean; + union { + u32 next_to_use; + u32 next_to_post; + }; + u32 ring_len; +}; + +#define LIBIE_CTLQ_MBX_ATQ_LEN GENMASK(9, 0) + +/* libie controlq descriptor qword0 details */ + +/* Flags sub-structure + * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 | + * |DD |CMP|ERR| * RSV * |FTYPE | *RSV* |RD |VFC|BUF| HOST_ID | + */ +#define LIBIE_CTLQ_DESC_FLAG_DD BIT(0) +#define LIBIE_CTLQ_DESC_FLAG_CMP BIT(1) +#define LIBIE_CTLQ_DESC_FLAG_ERR BIT(2) +#define LIBIE_CTLQ_DESC_FLAG_FTYPE_VM BIT(6) +#define LIBIE_CTLQ_DESC_FLAG_FTYPE_PF BIT(7) +#define LIBIE_CTLQ_DESC_FLAG_FTYPE GENMASK(7, 6) +#define LIBIE_CTLQ_DESC_FLAG_RD BIT(10) +#define LIBIE_CTLQ_DESC_FLAG_VFC BIT(11) +#define LIBIE_CTLQ_DESC_FLAG_BUF BIT(12) +#define LIBIE_CTLQ_DESC_FLAG_HOST_ID GENMASK(15, 13) + +#define LIBIE_CTLQ_DESC_FLAGS GENMASK(15, 0) +#define LIBIE_CTLQ_DESC_INFRA_OPCODE GENMASK_ULL(31, 16) +#define LIBIE_CTLQ_DESC_DATA_LEN GENMASK_ULL(47, 32) +#define LIBIE_CTLQ_DESC_HW_RETVAL GENMASK_ULL(63, 48) + +#define LIBIE_CTLQ_DESC_PFID_VFID GENMASK_ULL(63, 48) + +/* libie controlq descriptor qword1 details */ +#define LIBIE_CTLQ_DESC_VIRTCHNL_OPCODE GENMASK(27, 0) +#define LIBIE_CTLQ_DESC_VIRTCHNL_DESC_TYPE GENMASK_ULL(31, 28) +#define LIBIE_CTLQ_DESC_VIRTCHNL_MSG_RET_VAL GENMASK_ULL(63, 32) + +/* libie controlq descriptor qword2 details */ +#define LIBIE_CTLQ_DESC_MSG_PARAM0 GENMASK_ULL(31, 0) +#define LIBIE_CTLQ_DESC_SW_COOKIE GENMASK_ULL(47, 32) +#define LIBIE_CTLQ_DESC_VIRTCHNL_FLAGS GENMASK_ULL(63, 48) + +/* libie controlq descriptor qword3 details */ +#define LIBIE_CTLQ_DESC_DATA_ADDR_HIGH GENMASK_ULL(31, 0) +#define LIBIE_CTLQ_DESC_DATA_ADDR_LOW GENMASK_ULL(63, 32) + +/** + * struct libie_ctlq_desc - control queue descriptor format + * @qword0: flags, message opcode, data length etc + * @qword1: virtchnl opcode, descriptor type and return value + * @qword2: indirect message parameters + * @qword3: indirect message buffer address + */ +struct libie_ctlq_desc { + __le64 qword0; + __le64 qword1; + __le64 qword2; + __le64 qword3; +}; + +/** + * struct libie_ctlq_clean_params - cleaning parameters for Tx messages + * @rel_dma_mem: non-sleeping callback to put the DMA buffer after send + * @rel_ctx: additional context for release callback + * @ctlq: control queue information + * @num_msgs: number of messages to be cleaned + * @force: clean even if DD is not yet set, use only for final cleanup + */ +struct libie_ctlq_clean_params { + void (*rel_dma_mem)(const void *ctx, struct libie_cp_dma_mem *dma_mem); + const void *rel_ctx; + struct libie_ctlq_info *ctlq; + u16 num_msgs; + bool force; +}; + +/** + * libie_ctlq_release_rx_buf - Release Rx buffer for a specific control queue + * @rx_buf: Rx buffer to be freed + * + * Driver uses this function to post back the Rx buffer after the usage. + */ +static inline void libie_ctlq_release_rx_buf(struct kvec *rx_buf) +{ + netmem_ref netmem; + + if (!rx_buf->iov_base) + return; + + netmem = virt_to_netmem(rx_buf->iov_base); + page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false); +} + +int libie_ctlq_init(struct libie_ctlq_ctx *ctx, + const struct libie_ctlq_create_info *qinfo, u32 numq); +void libie_ctlq_deinit(struct libie_ctlq_ctx *ctx); + +struct libie_ctlq_info *libie_find_ctlq(struct libie_ctlq_ctx *ctx, + enum libie_ctlq_type type, + int id); + +u32 libie_ctlq_send_desc_avail(const struct libie_ctlq_info *ctlq); +void libie_ctlq_send(struct libie_ctlq_info *ctlq, u32 num_q_msg); +u32 libie_ctlq_send_clean(const struct libie_ctlq_clean_params *params); +u32 libie_ctlq_recv(struct libie_ctlq_info *ctlq, struct libie_ctlq_msg *msg, + u32 num_q_msg); + +int libie_ctlq_post_rx_buffs(struct libie_ctlq_info *ctlq); + +/* Only 8 bits are available in descriptor for Xn index */ +#define LIBIE_CTLQ_MAX_XN_ENTRIES 256 +#define LIBIE_CTLQ_XN_COOKIE_M GENMASK(15, 8) +#define LIBIE_CTLQ_XN_INDEX_M GENMASK(7, 0) + +/** + * enum libie_ctlq_xn_state - Transaction state of a virtchnl message + * @LIBIE_CTLQ_XN_IDLE: transaction is available to use + * @LIBIE_CTLQ_XN_WAITING: waiting for transaction to complete + * @LIBIE_CTLQ_XN_COMPLETED_SUCCESS: transaction completed with success + * @LIBIE_CTLQ_XN_COMPLETED_FAILED: transaction completed with failure + * @LIBIE_CTLQ_XN_ASYNC: asynchronous virtchnl message transaction type + * @LIBIE_CTLQ_XN_SHUTDOWN: transaction cannot be used anymore + */ +enum libie_ctlq_xn_state { + LIBIE_CTLQ_XN_IDLE = 0, + LIBIE_CTLQ_XN_WAITING, + LIBIE_CTLQ_XN_COMPLETED_SUCCESS, + LIBIE_CTLQ_XN_COMPLETED_FAILED, + LIBIE_CTLQ_XN_ASYNC, + LIBIE_CTLQ_XN_SHUTDOWN, +}; + +/** + * struct libie_ctlq_xn - structure representing a virtchnl transaction entry + * @resp_cb: non-sleeping callback to handle the response to an async message + * @xn_lock: lock to protect the transaction entry state + * @cmd_completion_event: wait until reply is received or xn is terminated + * @small_dma_mem: DMA memory for copying small send buffers from stack, + * is recycled when response is received or on timeout + * @send_dma_mem: DMA memory of send buffer + * @recv_mem: receive buffer + * @send_ctx: context for callback function + * @timeout_ms: Xn transaction timeout in msecs + * @timestamp: timestamp to record the Xn send + * @tx_msg: control queue Tx message slot to track small DMA usage + * @virtchnl_opcode: virtchnl command opcode used for Xn transaction + * @state: transaction state of a virtchnl message + * @cookie: unique message identifier, incremented every time the slot is used + * @index: index of the transaction entry + */ +struct libie_ctlq_xn { + void (*resp_cb)(void *ctx, struct kvec *mem, int status); + spinlock_t xn_lock; /* protects state */ + struct completion cmd_completion_event; + struct libie_cp_dma_mem small_dma_mem; + struct libie_cp_dma_mem send_dma_mem; + struct kvec recv_mem; + void *send_ctx; + u64 timeout_ms; + ktime_t timestamp; + struct libie_ctlq_msg *tx_msg; + u32 virtchnl_opcode; + enum libie_ctlq_xn_state state; + u8 cookie; + u8 index; +}; + +/** + * struct libie_ctlq_xn_manager - structure representing the array of virtchnl + * transaction entries + * @ctx: pointer to controlq context structure + * @free_xns_bm_lock: lock to protect the free Xn entries bit map + * @free_xns_bm: bitmap that represents the free Xn entries + * @ring: array of Xn entries + * @small_buff_pool: DMA pool for small send buffers + * @can_destroy: completion, triggered by the last released transaction + * @shutdown: shutdown process has been started, no new transactions allowed + */ +struct libie_ctlq_xn_manager { + struct libie_ctlq_ctx *ctx; + spinlock_t free_xns_bm_lock; /* get/check entries */ + DECLARE_BITMAP(free_xns_bm, LIBIE_CTLQ_MAX_XN_ENTRIES); + struct libie_ctlq_xn ring[LIBIE_CTLQ_MAX_XN_ENTRIES]; + struct dma_pool *small_buff_pool; + struct completion can_destroy; + bool shutdown; +}; + +/** + * struct libie_ctlq_xn_send_params - structure representing send Xn entry + * @resp_cb: non-sleeping callback to handle the response to an async message + * @rel_tx_buf: non-sleeping callback for freeing the send buffer + * @xnm: Xn manager to process Xn entries + * @ctlq: send control queue information + * @ctlq_msg: control queue message information + * @send_buf: buffer that carries outgoing message data, buffers larger than + * LIBIE_CP_TX_COPYBREAK bytes will always be consumed + * @recv_mem: receive buffer + * @send_ctx: context for callback function + * @timeout_ms: virtchnl transaction timeout in msecs + * @chnl_opcode: virtchnl message opcode + */ +struct libie_ctlq_xn_send_params { + void (*resp_cb)(void *ctx, struct kvec *mem, int status); + void (*rel_tx_buf)(const void *buf_va); + struct libie_ctlq_xn_manager *xnm; + struct libie_ctlq_info *ctlq; + struct libie_ctlq_msg *ctlq_msg; + struct kvec send_buf; + struct kvec recv_mem; + void *send_ctx; + u64 timeout_ms; + u32 chnl_opcode; +}; + +/** + * libie_cp_can_send_onstack - can a message be sent using a stack variable + * @size: ctlq data buffer size + * + * Return: %true if the message size is small enough for caller to pass + * an on-stack buffer, %false if kmalloc is needed + */ +static inline bool libie_cp_can_send_onstack(u32 size) +{ + return size <= LIBIE_CP_TX_COPYBREAK; +} + +/** + * struct libie_ctlq_xn_recv_params - request to receive xn responses + * @ctlq_msg_handler: handler for Rx messages with no matching xn (mandatory) + * @xnm: Xn manager to process Xn entries + * @ctlq: control queue information + * @budget: maximum number of messages to process + */ +struct libie_ctlq_xn_recv_params { + void (*ctlq_msg_handler)(struct libie_ctlq_ctx *ctx, + struct libie_ctlq_msg *msg); + struct libie_ctlq_xn_manager *xnm; + struct libie_ctlq_info *ctlq; + u32 budget; +}; + +/** + * struct libie_ctlq_xn_init_params - xn transaction manager parameters + * @cctlq_info: control queue information + * @ctx: pointer to controlq context structure + * @xnm: Xn manager to process Xn entries + * @num_qs: number of control queues to be initialized + */ +struct libie_ctlq_xn_init_params { + struct libie_ctlq_create_info *cctlq_info; + struct libie_ctlq_ctx *ctx; + struct libie_ctlq_xn_manager *xnm; + u32 num_qs; +}; + +int libie_ctlq_xn_init(struct libie_ctlq_xn_init_params *params); +void libie_ctlq_xn_deinit(struct libie_ctlq_xn_manager *xnm, + struct libie_ctlq_ctx *ctx); +void libie_ctlq_xn_shutdown(struct libie_ctlq_xn_manager *xnm); +int libie_ctlq_xn_send(struct libie_ctlq_xn_send_params *params); +u32 libie_ctlq_xn_recv(struct libie_ctlq_xn_recv_params *params); +u32 libie_ctlq_xn_send_clean(struct libie_ctlq_info *ctlq, + void (*rel_tx_buf)(const void *buf_va), + bool force); + +#endif /* __LIBIE_CONTROLQ_H */ diff --git a/include/linux/net/intel/libie/pci.h b/include/linux/net/intel/libie/pci.h new file mode 100644 index 000000000000..effd072c55c8 --- /dev/null +++ b/include/linux/net/intel/libie/pci.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef __LIBIE_PCI_H +#define __LIBIE_PCI_H + +#include + +/** + * struct libie_pci_mmio_region - structure for MMIO region info + * @list: used to add a MMIO region to the list of MMIO regions in + * libie_mmio_info + * @addr: virtual address of MMIO region start + * @offset: start offset of the MMIO region + * @size: size of the MMIO region + * @bar_idx: BAR index to which the MMIO region belongs to + */ +struct libie_pci_mmio_region { + struct list_head list; + void __iomem *addr; + resource_size_t offset; + resource_size_t size; + u16 bar_idx; +}; + +/** + * struct libie_mmio_info - contains list of MMIO regions + * @pdev: PCI device pointer + * @mmio_list: list of MMIO regions + */ +struct libie_mmio_info { + struct pci_dev *pdev; + struct list_head mmio_list; +}; + +#define libie_pci_map_mmio_region(mmio_info, offset, size, ...) \ + __libie_pci_map_mmio_region(mmio_info, offset, size, \ + COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__) + +#define libie_pci_get_mmio_addr(mmio_info, offset, ...) \ + __libie_pci_get_mmio_addr(mmio_info, offset, \ + COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__) + +bool __libie_pci_map_mmio_region(struct libie_mmio_info *mmio_info, + resource_size_t offset, resource_size_t size, + int num_args, ...); +void __iomem *__libie_pci_get_mmio_addr(struct libie_mmio_info *mmio_info, + resource_size_t offset, + int num_args, ...); +void libie_pci_unmap_all_mmio_regions(struct libie_mmio_info *mmio_info); +void libie_pci_unmap_fltr_regs(struct libie_mmio_info *mmio_info, + bool (*fltr)(struct libie_mmio_info *mmio_info, + struct libie_pci_mmio_region *reg)); +int libie_pci_init_dev(struct pci_dev *pdev); + +#endif /* __LIBIE_PCI_H */ diff --git a/include/linux/avf/virtchnl.h b/include/linux/net/intel/virtchnl.h similarity index 100% rename from include/linux/avf/virtchnl.h rename to include/linux/net/intel/virtchnl.h diff --git a/drivers/net/ethernet/intel/idpf/virtchnl2.h b/include/linux/net/intel/virtchnl2.h similarity index 100% rename from drivers/net/ethernet/intel/idpf/virtchnl2.h rename to include/linux/net/intel/virtchnl2.h diff --git a/drivers/net/ethernet/intel/idpf/virtchnl2_lan_desc.h b/include/linux/net/intel/virtchnl2_lan_desc.h similarity index 100% rename from drivers/net/ethernet/intel/idpf/virtchnl2_lan_desc.h rename to include/linux/net/intel/virtchnl2_lan_desc.h diff --git a/include/net/libeth/rx.h b/include/net/libeth/rx.h index 5d991404845e..0e736846c5e8 100644 --- a/include/net/libeth/rx.h +++ b/include/net/libeth/rx.h @@ -69,6 +69,7 @@ enum libeth_fqe_type { * @type: type of the buffers this queue has * @hsplit: flag whether header split is enabled * @xdp: flag indicating whether XDP is enabled + * @no_napi: the queue is not a data queue and does not have NAPI * @buf_len: HW-writeable length per each buffer * @nid: ID of the closest NUMA node with memory */ @@ -85,12 +86,13 @@ struct libeth_fq { enum libeth_fqe_type type:2; bool hsplit:1; bool xdp:1; + bool no_napi:1; u32 buf_len; int nid; }; -int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi); +int libeth_rx_fq_create(struct libeth_fq *fq, void *napi_dev); void libeth_rx_fq_destroy(struct libeth_fq *fq); /**