mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
Merge branch 'eth-mpnic-initial-support-for-meta-platforms-nic'
Daniel Zahka says: ==================== eth: mpnic: initial support for Meta Platforms NIC Meta Platforms NIC is the second-generation NIC from Meta. There are quite a few high-level similarities to fbnic, but all of the CSRs and descriptors have been reshuffled, and new features have been added. Since the device is sufficiently different from fbnic, and the design of our FW is minimal, we have decided a separate driver is appropriate. Major new features compared to fbnic will take a while to land. This series includes only the bare-bones Tx/Rx datapath. Our high-level patch sequence and plan for the upcoming series is: 1. Core datapath (this series) 2. Basic offloads, RSS, and counters 3. Firmware mailbox support 4. Base ethtool operations This patch series was prepared in part with these LLM prompts: https://github.com/kuba-moo/ai-prompts/blob/main/driver-upstreaming-prompts.md The prompts were used to distill our fully-featured OOT driver into this minimal Tx/Rx capable upstream driver. We then performed human review on the output to create this posting. Jakub has made the prompts public in case people would like to take a look. v1: https://lore.kernel.org/r/20260922-linux-mpnic-v1-0-236844f53072@gmail.com Signed-off-by: Daniel Zahka <daniel.zahka@gmail.com> ==================== Link: https://patch.msgid.link/20260924-linux-mpnic-v2-0-4badc9b58b9e@gmail.com Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni
commit
09c188c249
|
|
@ -35,4 +35,17 @@ config FBNIC
|
|||
To compile this driver as a module, choose M here. The module
|
||||
will be called fbnic. MSI-X interrupt support is required.
|
||||
|
||||
config MPNIC
|
||||
tristate "Meta Platforms Network Interface Controller"
|
||||
depends on 64BIT || COMPILE_TEST
|
||||
depends on !S390
|
||||
depends on PCI_MSI
|
||||
select PAGE_POOL
|
||||
help
|
||||
This driver supports the Meta Platforms Network Interface
|
||||
Controller.
|
||||
|
||||
To compile this driver as a module, choose M here. The module
|
||||
will be called mpnic.
|
||||
|
||||
endif # NET_VENDOR_META
|
||||
|
|
|
|||
|
|
@ -4,3 +4,4 @@
|
|||
#
|
||||
|
||||
obj-$(CONFIG_FBNIC) += fbnic/
|
||||
obj-$(CONFIG_MPNIC) += mpnic/
|
||||
|
|
|
|||
16
drivers/net/ethernet/meta/mpnic/Makefile
Normal file
16
drivers/net/ethernet/meta/mpnic/Makefile
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
# SPDX-License-Identifier: GPL-2.0
|
||||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
|
||||
#
|
||||
# Makefile for the Meta(R) Platforms Network Interface Controller
|
||||
#
|
||||
|
||||
obj-$(CONFIG_MPNIC) += mpnic.o
|
||||
|
||||
mpnic-y := \
|
||||
mpnic_init.o \
|
||||
mpnic_irq.o \
|
||||
mpnic_netdev.o \
|
||||
mpnic_pci.o \
|
||||
mpnic_txrx.o \
|
||||
# End of mpnic-y
|
||||
66
drivers/net/ethernet/meta/mpnic/mpnic.h
Normal file
66
drivers/net/ethernet/meta/mpnic/mpnic.h
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#ifndef _MPNIC_H_
|
||||
#define _MPNIC_H_
|
||||
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/io-64-nonatomic-lo-hi.h>
|
||||
#include <linux/types.h>
|
||||
|
||||
#include "mpnic_csr.h"
|
||||
|
||||
#define MPNIC_DRV_NAME "mpnic"
|
||||
|
||||
#define MPNIC_MAX_TXQS 1024u
|
||||
#define MPNIC_MAX_RXQS 1024u
|
||||
|
||||
/* misc IRQ entries are allocated before the completion queue IRQs */
|
||||
enum {
|
||||
MPNIC_FW_MSIX_ENTRY,
|
||||
MPNIC_NON_NAPI_VECTORS
|
||||
};
|
||||
|
||||
struct mpnic_dev {
|
||||
struct device *dev;
|
||||
struct net_device *netdev;
|
||||
|
||||
u32 __iomem *uc_addr0;
|
||||
|
||||
u16 num_irqs;
|
||||
|
||||
u64 dsn;
|
||||
u32 mps;
|
||||
u32 readrq;
|
||||
u8 relaxed_ord;
|
||||
};
|
||||
|
||||
u64 mpnic_rd64(struct mpnic_dev *mpd, u32 reg);
|
||||
|
||||
int mpnic_dev_init(struct mpnic_dev *mpd);
|
||||
|
||||
int mpnic_request_irq(struct mpnic_dev *mpd, int nr, irq_handler_t handler,
|
||||
unsigned long flags, const char *name, void *data);
|
||||
void mpnic_free_irq(struct mpnic_dev *mpd, int nr, void *data);
|
||||
void mpnic_free_irqs(struct mpnic_dev *mpd);
|
||||
int mpnic_alloc_irqs(struct mpnic_dev *mpd);
|
||||
|
||||
static inline void mpnic_wr64(struct mpnic_dev *mpd, u32 reg, u64 val)
|
||||
{
|
||||
u32 __iomem *csr = READ_ONCE(mpd->uc_addr0);
|
||||
|
||||
if (csr)
|
||||
writeq(val, csr + reg);
|
||||
}
|
||||
|
||||
static inline void mpnic_wrfl(struct mpnic_dev *mpd)
|
||||
{
|
||||
mpnic_rd64(mpd, MPNIC_BDQ_SPARE);
|
||||
}
|
||||
|
||||
static inline bool mpnic_present(struct mpnic_dev *mpd)
|
||||
{
|
||||
return !!READ_ONCE(mpd->uc_addr0);
|
||||
}
|
||||
|
||||
#endif /* _MPNIC_H_ */
|
||||
319
drivers/net/ethernet/meta/mpnic/mpnic_csr.h
Normal file
319
drivers/net/ethernet/meta/mpnic/mpnic_csr.h
Normal file
|
|
@ -0,0 +1,319 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#ifndef _MPNIC_CSR_H_
|
||||
#define _MPNIC_CSR_H_
|
||||
|
||||
#include <linux/bits.h>
|
||||
|
||||
#define CSR_BIT(nr) BIT_ULL(nr)
|
||||
#define CSR_GENMASK(h, l) GENMASK_ULL(h, l)
|
||||
|
||||
#define DESC_BIT(nr) BIT_ULL(nr)
|
||||
#define DESC_GENMASK(h, l) GENMASK_ULL(h, l)
|
||||
|
||||
/* Transmit Work Descriptor Format */
|
||||
#define MPNIC_TWD_FLAG_REQ_COMPLETION DESC_BIT(37)
|
||||
#define MPNIC_TWD_FLAG_DEST_MAC DESC_BIT(43)
|
||||
#define MPNIC_TWD_TYPE DESC_GENMASK(47, 46)
|
||||
enum {
|
||||
MPNIC_TWD_TYPE_META = 0,
|
||||
MPNIC_TWD_TYPE_AL = 2,
|
||||
MPNIC_TWD_TYPE_LAST_AL = 3,
|
||||
};
|
||||
|
||||
#define MPNIC_TWD_ADDR DESC_GENMASK(45, 0)
|
||||
#define MPNIC_TWD_LEN DESC_GENMASK(63, 48)
|
||||
|
||||
/* Tx Completion Descriptor Format */
|
||||
#define MPNIC_TCD_TYPE0_HEAD0 DESC_GENMASK(15, 0)
|
||||
#define MPNIC_TCD_DONE DESC_BIT(63)
|
||||
|
||||
/* Rx Buffer Descriptor Format */
|
||||
#define MPNIC_BD_DESC_ADDR DESC_GENMASK(39, 2)
|
||||
#define MPNIC_BD_DESC_ID DESC_GENMASK(57, 40)
|
||||
#define MPNIC_BD_DESC_BUF_SZ_LOG2 DESC_GENMASK(62, 58)
|
||||
|
||||
/* Rx Completion Queue Descriptors */
|
||||
#define MPNIC_RCD_TYPE DESC_GENMASK(62, 61)
|
||||
enum {
|
||||
MPNIC_RCD_TYPE_HDR_AL = 0,
|
||||
MPNIC_RCD_TYPE_PAY_AL = 1,
|
||||
MPNIC_RCD_TYPE_META = 3,
|
||||
};
|
||||
|
||||
#define MPNIC_RCD_DONE DESC_BIT(63)
|
||||
|
||||
#define MPNIC_RCD_HDR_SUBTYPE DESC_GENMASK(60, 59)
|
||||
enum {
|
||||
MPNIC_RCD_HDR_SUBTYPE_HDR = 2,
|
||||
};
|
||||
|
||||
/* Address/Length Completion Descriptors */
|
||||
#define MPNIC_RCD_AL_BUFF_OFF DESC_GENMASK(15, 0)
|
||||
#define MPNIC_RCD_AL_BUFF_ID DESC_GENMASK(33, 16)
|
||||
#define MPNIC_RCD_AL_BUFF_LEN DESC_GENMASK(47, 34)
|
||||
#define MPNIC_RCD_AL_PAGE_FIN DESC_BIT(53)
|
||||
|
||||
/* Metadata Completion Descriptors */
|
||||
#define MPNIC_RCD_META_ERR_MAC_EOP DESC_BIT(53)
|
||||
#define MPNIC_RCD_META_ERR_TRUNCATED_FRAME DESC_BIT(54)
|
||||
#define MPNIC_RCD_META_UNCORRECTABLE_ERR_MASK \
|
||||
(MPNIC_RCD_META_ERR_MAC_EOP | MPNIC_RCD_META_ERR_TRUNCATED_FRAME)
|
||||
|
||||
/* Common fields for all DESC_CFG CSRs */
|
||||
#define MPNIC_DESC_CFG_NUM_DESCS CSR_GENMASK(2, 0)
|
||||
#define MPNIC_DESC_CFG_START_ADDR CSR_GENMASK(19, 8)
|
||||
|
||||
/* Register Definitions
|
||||
*
|
||||
* The register file is addressed as an array of le32, so the byte address of
|
||||
* a register is 4 times the index below. Each register is listed with its
|
||||
* name, index and byte address.
|
||||
*
|
||||
* Name Index Address
|
||||
*****************************************************************************/
|
||||
|
||||
/* NIC_CORE_TDF */
|
||||
#define MPNIC_TWQ_CTL(i, j) (0x0 + 1024 * (i) + 2 * (j))
|
||||
/* 0x0 */
|
||||
#define MPNIC_TWQ_CTL_RESET CSR_BIT(0)
|
||||
#define MPNIC_TWQ_CTL_ENABLE CSR_BIT(1)
|
||||
#define MPNIC_TWQ_TAIL(i, j) (0x4 + 1024 * (i) + 2 * (j))
|
||||
/* 0x10 */
|
||||
#define MPNIC_TWQ_SIZE(i, j) (0x10 + 1024 * (i) + 2 * (j))
|
||||
/* 0x40 */
|
||||
#define MPNIC_TWQ_SIZE_SIZE CSR_GENMASK(3, 0)
|
||||
#define MPNIC_TWQ_BASE_ADDR(i, j) (0x1c + 1024 * (i) + 2 * (j))
|
||||
/* 0x70 */
|
||||
|
||||
/* NIC_CORE_TCM */
|
||||
#define MPNIC_TCQ_CTL(i) (0x80 + 1024 * (i)) /* 0x200 */
|
||||
#define MPNIC_TCQ_CTL_RESET CSR_BIT(0)
|
||||
#define MPNIC_TCQ_CTL_ENABLE CSR_BIT(1)
|
||||
#define MPNIC_TCQ_BASE_ADDR(i) (0x86 + 1024 * (i)) /* 0x218 */
|
||||
#define MPNIC_TCQ_HEAD(i) (0x8e + 1024 * (i)) /* 0x238 */
|
||||
#define MPNIC_TCQ_SIZE(i) (0x94 + 1024 * (i)) /* 0x250 */
|
||||
#define MPNIC_TCQ_SIZE_SIZE CSR_GENMASK(4, 0)
|
||||
|
||||
/* NIC_CORE_TIM */
|
||||
#define MPNIC_TIM_CTL1(i) (0xc0 + 1024 * (i)) /* 0x300 */
|
||||
#define MPNIC_TIM_CTL1_UPD_IGN_LONG_EVENT_CNT CSR_BIT(48)
|
||||
#define MPNIC_TIM_CTL1_UPD_IGN_LONG_TIME_CNT CSR_BIT(49)
|
||||
#define MPNIC_TIM_CTL1_UPD_IGN_SHORT_TIME_CNT CSR_BIT(50)
|
||||
#define MPNIC_TIM_CTL1_MASK CSR_BIT(51)
|
||||
#define MPNIC_TIM_CTL1_MASK_EN CSR_BIT(52)
|
||||
#define MPNIC_TIM_CTL1_TRIGGER CSR_BIT(53)
|
||||
#define MPNIC_TIM_INTR_MASK(i) (0xc8 + 1024 * (i)) /* 0x320 */
|
||||
#define MPNIC_TIM_INTR_MASK_MASK CSR_BIT(0)
|
||||
|
||||
/* NIC_CORE_RBP */
|
||||
#define MPNIC_BDQ_CTL(i) (0x200 + 1024 * (i)) /* 0x800 */
|
||||
#define MPNIC_BDQ_CTL_RESET CSR_BIT(0)
|
||||
#define MPNIC_BDQ_CTL_ENABLE CSR_BIT(1)
|
||||
#define MPNIC_BDQ_CTL_ENABLE_PPQ CSR_BIT(3)
|
||||
#define MPNIC_HPQ_TAIL(i) (0x202 + 1024 * (i)) /* 0x808 */
|
||||
#define MPNIC_PPQ_TAIL(i) (0x204 + 1024 * (i)) /* 0x810 */
|
||||
#define MPNIC_HPQ_SIZE(i) (0x20a + 1024 * (i)) /* 0x828 */
|
||||
#define MPNIC_HPQ_SIZE_SIZE CSR_GENMASK(4, 0)
|
||||
#define MPNIC_PPQ_SIZE(i) (0x20c + 1024 * (i)) /* 0x830 */
|
||||
#define MPNIC_PPQ_SIZE_SIZE CSR_GENMASK(4, 0)
|
||||
#define MPNIC_HPQ_BASE_ADDR(i) (0x216 + 1024 * (i)) /* 0x858 */
|
||||
#define MPNIC_PPQ_BASE_ADDR(i) (0x218 + 1024 * (i)) /* 0x860 */
|
||||
|
||||
/* NIC_CORE_RCM */
|
||||
#define MPNIC_RCQ_CTL(i) (0x280 + 1024 * (i)) /* 0xa00 */
|
||||
#define MPNIC_RCQ_CTL_RESET CSR_BIT(0)
|
||||
#define MPNIC_RCQ_CTL_ENABLE CSR_BIT(1)
|
||||
#define MPNIC_RCQ_BASE_ADDR(i) (0x286 + 1024 * (i)) /* 0xa18 */
|
||||
#define MPNIC_RCQ_HEAD(i) (0x28e + 1024 * (i)) /* 0xa38 */
|
||||
#define MPNIC_RCQ_SIZE(i) (0x294 + 1024 * (i)) /* 0xa50 */
|
||||
#define MPNIC_RCQ_SIZE_SIZE CSR_GENMASK(4, 0)
|
||||
|
||||
/* NIC_CORE_RIM */
|
||||
#define MPNIC_RIM_INTR_MASK(i) (0x2c8 + 1024 * (i)) /* 0xb20 */
|
||||
#define MPNIC_RIM_INTR_MASK_MASK CSR_BIT(0)
|
||||
|
||||
/* NIC_CORE_TIM_PRV */
|
||||
#define MPNIC_TIM_CTL(i) (0x100100 + 1024 * (i)) /* 0x400400 */
|
||||
|
||||
/* NIC_CORE_RDE */
|
||||
#define MPNIC_RDE_CFG(i) (0x10021c + 1024 * (i)) /* 0x400870 */
|
||||
#define MPNIC_RDE_CFG_MIN_TAIL_ROOM CSR_GENMASK(9, 0)
|
||||
#define MPNIC_RDE_CFG_MIN_HEAD_ROOM CSR_GENMASK(18, 10)
|
||||
#define MPNIC_RDE_CFG_MAX_HEADER_BYTES CSR_GENMASK(45, 32)
|
||||
|
||||
/* NIC_CORE_RIM_PRV */
|
||||
#define MPNIC_RIM_CTL(i) (0x100280 + 1024 * (i)) /* 0x400a00 */
|
||||
|
||||
/* NIC_CORE_RBP_HP_GLBL */
|
||||
#define MPNIC_HPQ_IDLE(i) (0x420000 + 2 * (i)) /* 0x1080000 */
|
||||
#define MPNIC_HPQ_IDLE_CNT 16
|
||||
#define MPNIC_PPQ_IDLE(i) (0x420060 + 2 * (i)) /* 0x1080180 */
|
||||
#define MPNIC_PPQ_IDLE_CNT 16
|
||||
#define MPNIC_BDQ_GLBL_CTL0 0x420080 /* 0x1080200 */
|
||||
#define MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE CSR_GENMASK(26, 18)
|
||||
#define MPNIC_BDQ_GLBL_CTL0_PREFETCH_SPACE_THRESH \
|
||||
CSR_GENMASK(42, 32)
|
||||
#define MPNIC_RDE_CTL 0x420082 /* 0x1080208 */
|
||||
#define MPNIC_RDE_CTL_HPQ_DROP_THRESHOLD CSR_GENMASK(10, 0)
|
||||
#define MPNIC_RDE_CTL_PPQ_DROP_THRESHOLD CSR_GENMASK(21, 11)
|
||||
#define MPNIC_RDE_CTL_HPQ_LOCAL_DROP_THRESHOLD CSR_GENMASK(42, 32)
|
||||
#define MPNIC_RDE_CTL_PPQ_LOCAL_DROP_THRESHOLD CSR_GENMASK(53, 43)
|
||||
#define MPNIC_BDQ_MEM_INIT_REQ 0x42013a /* 0x10804e8 */
|
||||
#define MPNIC_BDQ_MEM_INIT_DONE 0x42013c /* 0x10804f0 */
|
||||
#define MPNIC_BDQ_SPARE 0x42013e /* 0x10804f8 */
|
||||
#define MPNIC_HPQ_DESC_CFG(i) (0x420140 + 2 * (i)) /* 0x1080500 */
|
||||
#define MPNIC_PPQ_DESC_CFG(i) (0x420940 + 2 * (i)) /* 0x1082500 */
|
||||
|
||||
/* NIC_CORE_RDE_GLBL */
|
||||
#define MPNIC_RDE_MEM_INIT_REQ 0x4240e6 /* 0x1090398 */
|
||||
#define MPNIC_RDE_MEM_INIT_DONE 0x4240e8 /* 0x10903a0 */
|
||||
|
||||
/* NIC_CORE_RCM_GLBL */
|
||||
#define MPNIC_RCQ_IDLE(i) (0x42505e + 2 * (i)) /* 0x1094178 */
|
||||
#define MPNIC_RCQ_IDLE_CNT 16
|
||||
#define MPNIC_RCM_MEM_INIT_REQ 0x42507e /* 0x10941f8 */
|
||||
#define MPNIC_RCM_MEM_INIT_DONE 0x425080 /* 0x1094200 */
|
||||
|
||||
/* NIC_CORE_RNI_GLBL */
|
||||
#define MPNIC_RNI_RBP_CTL 0x427000 /* 0x109c000 */
|
||||
#define MPNIC_RNI_RDE_CTL 0x427002 /* 0x109c008 */
|
||||
#define MPNIC_RNI_RDE_CTL_MPS CSR_GENMASK(1, 0)
|
||||
#define MPNIC_RNI_RDE_CTL_CLS CSR_GENMASK(3, 2)
|
||||
#define MPNIC_RNI_RCM_CTL 0x427004 /* 0x109c010 */
|
||||
|
||||
/* NIC_CORE_TDF_GLBL */
|
||||
#define MPNIC_TWQ_IDLE(i) (0x428042 + 2 * (i)) /* 0x10a0108 */
|
||||
#define MPNIC_TWQ_IDLE_CNT 32
|
||||
#define MPNIC_TWQ_DEF_PRI_TWD 0x428082 /* 0x10a0208 */
|
||||
#define MPNIC_TDF_MEM_INIT_REQ 0x42813a /* 0x10a04e8 */
|
||||
#define MPNIC_TDF_MEM_INIT_DONE 0x42813c /* 0x10a04f0 */
|
||||
#define MPNIC_TDF_DESC_CFG(i) (0x428140 + 2 * (i)) /* 0x10a0500 */
|
||||
|
||||
/* NIC_CORE_TQS_GLBL */
|
||||
#define MPNIC_TQS_GLBL_CTL0 0x42a000 /* 0x10a8000 */
|
||||
#define MPNIC_TQS_GLBL_CTL0_TWD_ERROR_CHECK_EN CSR_BIT(2)
|
||||
#define MPNIC_TQS_GLBL_P0_0 0x42a002 /* 0x10a8008 */
|
||||
#define MPNIC_TQS_GLBL_P0_0_TXB_MAX_CRDTS_0 CSR_GENMASK(63, 48)
|
||||
#define MPNIC_TQS_GLBL_P0_1 0x42a004 /* 0x10a8010 */
|
||||
#define MPNIC_TQS_GLBL_BMC 0x42a012 /* 0x10a8048 */
|
||||
#define MPNIC_TQS_GLBL_BMC_TXB_MAX_CRDTS CSR_GENMASK(15, 0)
|
||||
#define MPNIC_TQS_SLOWDOWN_CTL 0x42a026 /* 0x10a8098 */
|
||||
#define MPNIC_TQS_SLOWDOWN_CTL_ENABLE CSR_BIT(6)
|
||||
#define MPNIC_TQS_MTU_CTL0 0x42a030 /* 0x10a80c0 */
|
||||
#define MPNIC_TQS_MTU_CTL1 0x42a032 /* 0x10a80c8 */
|
||||
#define MPNIC_TQS_IDLE(i) (0x42a040 + 2 * (i)) /* 0x10a8100 */
|
||||
#define MPNIC_TQS_IDLE_CNT 32
|
||||
#define MPNIC_TQS_SET_P0_MAP0(i) (0x42a082 + 2 * (i)) /* 0x10a8208 */
|
||||
#define MPNIC_TQS_SET_P0_MAP1(i) (0x42a092 + 2 * (i)) /* 0x10a8248 */
|
||||
#define MPNIC_TQS_GLBL_SHAPING 0x42a108 /* 0x10a8420 */
|
||||
#define MPNIC_TQS_GLBL_SHAPING_DISABLE CSR_BIT(0)
|
||||
#define MPNIC_TQS_ARB_CTL 0x42a122 /* 0x10a8488 */
|
||||
#define MPNIC_TQS_ARB_CTL_SET_CRDT_BUCKET_EN CSR_BIT(9)
|
||||
#define MPNIC_TQS_ARB_CTL_SET_IMM_DECR_EN CSR_BIT(8)
|
||||
#define MPNIC_TQS_ARB_CTL_GROUP_CRDT_BUCKET_EN CSR_BIT(5)
|
||||
#define MPNIC_TQS_ARB_CTL_GROUP_IMM_DECR_EN CSR_BIT(4)
|
||||
#define MPNIC_TQS_ARB_CTL_QUEUE_CRDT_BUCKET_EN CSR_BIT(1)
|
||||
#define MPNIC_TQS_ARB_CTL_QUEUE_IMM_DECR_EN CSR_BIT(0)
|
||||
#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0 \
|
||||
0x42a124 /* 0x10a8490 */
|
||||
#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_QUEUE CSR_GENMASK(19, 0)
|
||||
#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_GROUP CSR_GENMASK(59, 32)
|
||||
#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1 \
|
||||
0x42a126 /* 0x10a8498 */
|
||||
#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_SET CSR_GENMASK(27, 0)
|
||||
#define MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_PORT CSR_GENMASK(63, 32)
|
||||
#define MPNIC_TQS_SRAM_INIT_CTL 0x42a128 /* 0x10a84a0 */
|
||||
#define MPNIC_TQS_SRAM_INIT_CTL_QUANTUM CSR_GENMASK(31, 20)
|
||||
#define MPNIC_TQS_SRAM_INIT_CTL_INIT CSR_BIT(32)
|
||||
#define MPNIC_TQS_GROUP_INIT_CTL 0x42a12a /* 0x10a84a8 */
|
||||
#define MPNIC_TQS_GROUP_INIT_CTL_QUANTUM CSR_GENMASK(51, 32)
|
||||
#define MPNIC_TQS_GROUP_INIT_CTL_INIT CSR_BIT(52)
|
||||
#define MPNIC_TQS_SET_INIT_CTL 0x42a12c /* 0x10a84b0 */
|
||||
#define MPNIC_TQS_SET_INIT_CTL_QUANTUM CSR_GENMASK(51, 32)
|
||||
#define MPNIC_TQS_SET_INIT_CTL_INIT CSR_BIT(52)
|
||||
#define MPNIC_TQS_PORT_INIT_CTL 0x42a12e /* 0x10a84b8 */
|
||||
#define MPNIC_TQS_PORT_INIT_CTL_QUANTUM CSR_GENMASK(55, 32)
|
||||
#define MPNIC_TQS_PORT_INIT_CTL_INIT CSR_BIT(56)
|
||||
#define MPNIC_TQS_SRAM_STS 0x42a130 /* 0x10a84c0 */
|
||||
#define MPNIC_TQS_PORT_CTL(i) (0x42a1e4 + 2 * (i)) /* 0x10a8790 */
|
||||
|
||||
/* NIC_CORE_TDE_GLBL */
|
||||
#define MPNIC_TDE_IDLE(i) (0x42b000 + 2 * (i)) /* 0x10ac000 */
|
||||
#define MPNIC_TDE_IDLE_CNT 32
|
||||
#define MPNIC_TDE_MEM_INIT_REQ 0x42b1ee /* 0x10ac7b8 */
|
||||
#define MPNIC_TDE_MEM_INIT_DONE 0x42b1f0 /* 0x10ac7c0 */
|
||||
|
||||
/* NIC_CORE_TCM_GLBL */
|
||||
#define MPNIC_TCQ_IDLE(i) (0x42c09e + 2 * (i)) /* 0x10b0278 */
|
||||
#define MPNIC_TCQ_IDLE_CNT 16
|
||||
#define MPNIC_TCM_MEM_INIT_REQ 0x42c0be /* 0x10b02f8 */
|
||||
#define MPNIC_TCM_MEM_INIT_DONE 0x42c0c0 /* 0x10b0300 */
|
||||
|
||||
/* NIC_CORE_TNI_GLBL */
|
||||
#define MPNIC_TNI_GLBL_TDF_CTL 0x42e000 /* 0x10b8000 */
|
||||
#define MPNIC_TNI_GLBL_TDF_CTL_MRRS CSR_GENMASK(2, 0)
|
||||
#define MPNIC_TNI_GLBL_TDF_CTL_CLS CSR_GENMASK(5, 3)
|
||||
#define MPNIC_TNI_GLBL_TDE_CTL 0x42e002 /* 0x10b8008 */
|
||||
#define MPNIC_TNI_GLBL_TCM_CTL 0x42e004 /* 0x10b8010 */
|
||||
|
||||
/* NIC_CORE_TXB */
|
||||
#define MPNIC_TXB_PORT_CONFIG 0x600000 /* 0x1800000 */
|
||||
#define MPNIC_TXB_PORT_CONFIG_PORT_MODE CSR_GENMASK(15, 13)
|
||||
#define MPNIC_TXB_BMC 0x600122 /* 0x1800488 */
|
||||
#define MPNIC_TXB_P0(i) (0x600124 + 2 * (i)) /* 0x1800490 */
|
||||
#define MPNIC_TXB_P0_CNT 17
|
||||
#define MPNIC_TXB_P0_THRESH(i) (0x60025e + 2 * (i)) /* 0x1800978 */
|
||||
#define MPNIC_TXB_P0_ARB_WEIGHTS(i) (0x600378 + 2 * (i)) /* 0x1800de0 */
|
||||
|
||||
/* NIC_CORE_RXB */
|
||||
#define MPNIC_RXB_MEM_INIT_REQ 0x620002 /* 0x1880008 */
|
||||
#define MPNIC_RXB_MEM_INIT_DONE 0x620004 /* 0x1880010 */
|
||||
#define MPNIC_RXB_PORT_CFG(i) (0x620006 + 2 * (i)) /* 0x1880018 */
|
||||
#define MPNIC_RXB_PORT_CFG_FCS_STRIP_MODE CSR_GENMASK(22, 22)
|
||||
enum {
|
||||
MPNIC_FCS_MODE_KEEP = 0,
|
||||
MPNIC_FCS_MODE_STRIP = 1,
|
||||
};
|
||||
|
||||
#define MPNIC_RXB_PORT_CLASS_CFG(i) (0x620020 + 2 * (i)) /* 0x1880080 */
|
||||
#define MPNIC_RXB_PORT_CLASS_CFG_DEFAULT_L2_ACTION \
|
||||
CSR_GENMASK(0, 0)
|
||||
enum {
|
||||
MPNIC_L2_ACTION_DROP = 0,
|
||||
MPNIC_L2_ACTION_PASS = 1,
|
||||
};
|
||||
|
||||
#define MPNIC_RXB_TC_CRDTS(i) (0x620418 + 2 * (i)) /* 0x1881060 */
|
||||
#define MPNIC_RXB_POOL_COMMON_CRDTS(i) (0x620458 + 2 * (i)) /* 0x1881160 */
|
||||
#define MPNIC_RXB_COMMON_CRDT_CTRL_TC(i) \
|
||||
(0x620472 + 2 * (i)) /* 0x18811c8 */
|
||||
#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_THRESH CSR_GENMASK(15, 0)
|
||||
#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_TC_EN CSR_BIT(29)
|
||||
#define MPNIC_RXB_COMMON_CRDT_CTRL_TC_MAX_CRDTS CSR_GENMASK(47, 32)
|
||||
#define MPNIC_RXB_HOST_DROP_THRESH(i) (0x6205b0 + 2 * (i)) /* 0x18816c0 */
|
||||
#define MPNIC_RXB_BMC_CRDTS(i) (0x6205d0 + 2 * (i)) /* 0x1881740 */
|
||||
|
||||
/* NIC_CORE_RPC */
|
||||
#define MPNIC_RPC_MEM_INIT_REQ 0x780440 /* 0x1e01100 */
|
||||
#define MPNIC_RPC_MEM_INIT_DONE 0x780442 /* 0x1e01108 */
|
||||
|
||||
/* NIC_CORE_ROF */
|
||||
#define MPNIC_RSC_GLOBAL_CONF 0x7e2002 /* 0x1f88008 */
|
||||
#define MPNIC_RSC_GLOBAL_CONF_RSC_DISABLE CSR_BIT(0)
|
||||
|
||||
/* NIC_CORE_TOF */
|
||||
#define MPNIC_TOF_TCAM_DEST_REMAP 0x7e3022 /* 0x1f8c088 */
|
||||
|
||||
/* PEMO_WRAPPER */
|
||||
#define MPNIC_OB_ATTR_RO CSR_BIT(1)
|
||||
#define MPNIC_OB_ATTR_TDE_H 0x9a000e /* 0x2680038 */
|
||||
#define MPNIC_OB_ATTR_TDE_P 0x9a0010 /* 0x2680040 */
|
||||
#define MPNIC_OB_ATTR_TDF 0x9a0012 /* 0x2680048 */
|
||||
#define MPNIC_OB_ATTR_RBP_HPQ 0x9a0014 /* 0x2680050 */
|
||||
#define MPNIC_OB_ATTR_RBP_PPQ 0x9a0016 /* 0x2680058 */
|
||||
#define MPNIC_OB_ATTR_RDE_H 0x9a0018 /* 0x2680060 */
|
||||
#define MPNIC_OB_ATTR_RDE_P 0x9a001a /* 0x2680068 */
|
||||
|
||||
#endif /* _MPNIC_CSR_H_ */
|
||||
556
drivers/net/ethernet/meta/mpnic/mpnic_init.c
Normal file
556
drivers/net/ethernet/meta/mpnic/mpnic_init.c
Normal file
|
|
@ -0,0 +1,556 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/bits.h>
|
||||
#include <linux/cache.h>
|
||||
#include <linux/if_ether.h>
|
||||
#include <linux/iopoll.h>
|
||||
#include <linux/log2.h>
|
||||
#include <linux/sizes.h>
|
||||
|
||||
#include "mpnic.h"
|
||||
|
||||
#define MPNIC_MEM_INIT_POLL_US 500
|
||||
#define MPNIC_MEM_INIT_TO_US 5000
|
||||
|
||||
/* BDQ mem init:
|
||||
* bit 1: fifo_wrptr_mem
|
||||
* bit 0: fifo_rdptr_mem
|
||||
*/
|
||||
#define MPNIC_MEM_INIT_BDQ_VAL 0x3
|
||||
|
||||
/* RCM mem init:
|
||||
* bit 3: cq_base_addr
|
||||
* bit 2: cd_fifo_rptr_stats
|
||||
* bit 1: cd_fifo_wptr_stats
|
||||
* bit 0: cq_head_ptr_stats
|
||||
*/
|
||||
#define MPNIC_MEM_INIT_RCM_VAL 0xf
|
||||
|
||||
/* RDE mem init, bits 0-18. Bits 0-12 are the per-queue packet, error and
|
||||
* drop counters, bits 13-18 the two context memories of each of the HPQ,
|
||||
* PPQ and SPQ descriptor prefetchers.
|
||||
*/
|
||||
#define MPNIC_MEM_INIT_RDE_VAL 0x7ffff
|
||||
|
||||
/* RPC mem init, bit 0 covers the whole classifier. */
|
||||
#define MPNIC_MEM_INIT_RPC_VAL 0x1
|
||||
|
||||
/* TCM mem init:
|
||||
* bit 3: cq_head_ptr_stats
|
||||
* bit 2: cd_fifo_wptr_stats
|
||||
* bit 1: cd_fifo_rptr_stats
|
||||
* bit 0: cq_base_addr
|
||||
*/
|
||||
#define MPNIC_MEM_INIT_TCM_VAL 0xf
|
||||
|
||||
/* TDE mem init:
|
||||
* bit 1: stats mem
|
||||
* bit 0: dma_head_ptr SRAM
|
||||
*/
|
||||
#define MPNIC_MEM_INIT_TDE_VAL 0x3
|
||||
|
||||
/* TDF mem init, bit 0 covers the descriptor fetch SRAM. */
|
||||
#define MPNIC_MEM_INIT_TDF_VAL 0x1
|
||||
|
||||
/* RXB mem init, bit 0 covers the DMAC TCAM statistics RAM. */
|
||||
#define MPNIC_MEM_INIT_RXB_VAL 0x1
|
||||
|
||||
/* TQS arbiter SRAM init done, one bit per DWRR level. */
|
||||
#define MPNIC_TQS_ARB_INIT_VAL 0xf
|
||||
|
||||
/* On-chip SRAM allocated to each queue for descriptor fetch, in units of
|
||||
* descriptors. Valid values are 64, 128, 256, 512 and 1024.
|
||||
*
|
||||
* The partition sizes are chosen for the maximum number of queues the
|
||||
* device supports, so they do not have to be adjusted when the active
|
||||
* queue count changes:
|
||||
*
|
||||
* BDQ: 1 MiB / (8 B/DESC) / (1024 HPQ + 1024 PPQ) = 64 DESC/QUEUE
|
||||
* TWQ: 2 MiB / (8 B/DESC) / (1024 TXQ * 2 TWQ) = 128 DESC/QUEUE
|
||||
*/
|
||||
#define MPNIC_BDQ_SRAM_DESCS 64u
|
||||
#define MPNIC_TDF_SRAM_DESCS 128u
|
||||
|
||||
/* A total of 1 MiB worth of Tx credits is available, in units of 128 B.
|
||||
* The BMC to MAC and host to BMC paths each get a guaranteed share of them
|
||||
* whether or not anything is routed their way, everything else goes to
|
||||
* MAC TC0.
|
||||
*/
|
||||
#define MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL 800
|
||||
#define MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL \
|
||||
(SZ_1M / 128 - 2 * MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL)
|
||||
|
||||
/* The recommended lower bound for the TXB threshold is 80, based on a 10K
|
||||
* MTU. Round up by 25% to stay on the defensive side. The same reasoning
|
||||
* applies to the arbitration weight, which has to exceed the full packet
|
||||
* size.
|
||||
*/
|
||||
#define MPNIC_TXB_INIT_BMC_THRESH 100
|
||||
#define MPNIC_TXB_INIT_P0_THRESH 100
|
||||
#define MPNIC_TXB_INIT_P0_ARB_WEIGHTS 0x64
|
||||
|
||||
/* RXB host drop threshold in units of 128 B beats. Packets targeting a
|
||||
* queue are dropped when the available credits fall below it. 80 beats is
|
||||
* 10 KB, which is also the largest frame the device is configured for.
|
||||
*/
|
||||
#define MPNIC_RXB_INIT_HOST_DROP_THRESH 0x50
|
||||
|
||||
/* A total of 8 MiB of Rx buffer is available. The recommended per-TC pool
|
||||
* for a 800G configuration is 420 KB with all 8 TCs enabled. Only one TC
|
||||
* is in use, so give it 8 * 420 KB (in units of 128 B) and push the rest
|
||||
* to the common pool. Start drawing from the common pool as soon as the
|
||||
* TC0 credits fall below one max sized frame. The BMC keeps a small
|
||||
* reserve of its own whether or not the host talks to it.
|
||||
*/
|
||||
#define MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 (0xd20 * 8)
|
||||
#define MPNIC_RXB_INIT_BMC_CRDTS 0x20
|
||||
#define MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH \
|
||||
(SZ_8M / 128 - MPNIC_RXB_INIT_POOL_TC_CRDTS_P0 - \
|
||||
MPNIC_RXB_INIT_BMC_CRDTS)
|
||||
#define MPNIC_RXB_INIT_COMMON_CRDT_THRSH 0x50
|
||||
|
||||
/* TXB port mode selects the number of active MAC ports for Tx buffer
|
||||
* credit distribution and arbitration. The hardware only supports single,
|
||||
* dual and quad port, encoded as b'001, b'010 and b'100.
|
||||
*/
|
||||
#define MPNIC_TXB_PORT_MODE_SINGLE 1
|
||||
|
||||
/* The MAC traffic classes start at index 8 in the TXB arrays, the BMC
|
||||
* sits above them.
|
||||
*/
|
||||
#define MPNIC_TXB_TC_IDX_MAC_0 8
|
||||
#define MPNIC_TXB_TC_IDX_BMC 16
|
||||
|
||||
/* Largest frame the Tx queue scheduler will pass through. Anything above
|
||||
* it gets truncated.
|
||||
*/
|
||||
#define MPNIC_TQS_MTU_CTL0_MAX 0x2800
|
||||
|
||||
/* The unit of the DWRR quantum is 256 B. It has to be large enough for at
|
||||
* least 11 MTUs to be transmitted in one quantum; use 15 for headroom.
|
||||
*/
|
||||
#define MPNIC_TQS_DWRR_INIT_QUANTUM (15 * MPNIC_TQS_MTU_CTL0_MAX / 256)
|
||||
|
||||
/* Lower bound on the credit available to a queue, group, set or port
|
||||
* before the scheduler stops issuing requests for it. 15 MTUs, matching
|
||||
* the DWRR quantum above.
|
||||
*/
|
||||
#define MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH (15 * MPNIC_TQS_MTU_CTL0_MAX)
|
||||
|
||||
/* The TQS view of the TXB partitioning between the MAC and the BMC, in
|
||||
* units of 1 KiB. It has to match the TXB private credits above.
|
||||
*/
|
||||
#define MPNIC_TQS_GLBL_TXB_CRDT_UNIT SZ_1K
|
||||
#define MPNIC_TQS_GLBL_TXB_CRDT_BMC \
|
||||
(MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL * 128 / MPNIC_TQS_GLBL_TXB_CRDT_UNIT)
|
||||
#define MPNIC_TQS_GLBL_TXB_CRDT_MAC \
|
||||
(MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL * 128 / \
|
||||
MPNIC_TQS_GLBL_TXB_CRDT_UNIT)
|
||||
|
||||
struct mpnic_init_poll {
|
||||
u64 exp_val;
|
||||
u32 addr;
|
||||
};
|
||||
|
||||
struct mpnic_poll_state {
|
||||
int poll_idx;
|
||||
u64 val;
|
||||
};
|
||||
|
||||
static void mpnic_tdf_glbl_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
/* Default metadata descriptor, used for frames the driver did not
|
||||
* prepend one to.
|
||||
*/
|
||||
mpnic_wr64(mpd, MPNIC_TWQ_DEF_PRI_TWD, MPNIC_TWD_FLAG_REQ_COMPLETION);
|
||||
}
|
||||
|
||||
static void mpnic_txb_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
int i;
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_TXB_BMC, MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL);
|
||||
|
||||
/* Zero the private credits of every traffic class, then hand the
|
||||
* unreserved ones to MAC TC0.
|
||||
*/
|
||||
for (i = 0; i < MPNIC_TXB_P0_CNT; i++)
|
||||
mpnic_wr64(mpd, MPNIC_TXB_P0(i), 0);
|
||||
mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_MAC_0),
|
||||
MPNIC_TXB_P0_MAC_PVT_CRDT_INIT_VAL);
|
||||
mpnic_wr64(mpd, MPNIC_TXB_P0(MPNIC_TXB_TC_IDX_BMC),
|
||||
MPNIC_TXB_BMC_PVT_CRDT_INIT_VAL);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_BMC),
|
||||
MPNIC_TXB_INIT_BMC_THRESH);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_TXB_P0_THRESH(MPNIC_TXB_TC_IDX_MAC_0),
|
||||
MPNIC_TXB_INIT_P0_THRESH);
|
||||
mpnic_wr64(mpd, MPNIC_TXB_P0_ARB_WEIGHTS(MPNIC_TXB_TC_IDX_MAC_0),
|
||||
MPNIC_TXB_INIT_P0_ARB_WEIGHTS);
|
||||
mpnic_wr64(mpd, MPNIC_TXB_PORT_CONFIG,
|
||||
FIELD_PREP(MPNIC_TXB_PORT_CONFIG_PORT_MODE,
|
||||
MPNIC_TXB_PORT_MODE_SINGLE));
|
||||
}
|
||||
|
||||
static void mpnic_rxb_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
/* Accept all packets that miss dmac tcam, until l2 filtering is
|
||||
* implemented.
|
||||
*/
|
||||
mpnic_wr64(mpd, MPNIC_RXB_PORT_CLASS_CFG(0),
|
||||
FIELD_PREP(MPNIC_RXB_PORT_CLASS_CFG_DEFAULT_L2_ACTION,
|
||||
MPNIC_L2_ACTION_PASS));
|
||||
mpnic_wr64(mpd, MPNIC_RXB_PORT_CFG(0),
|
||||
FIELD_PREP(MPNIC_RXB_PORT_CFG_FCS_STRIP_MODE,
|
||||
MPNIC_FCS_MODE_STRIP));
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_RXB_HOST_DROP_THRESH(0),
|
||||
MPNIC_RXB_INIT_HOST_DROP_THRESH);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_RXB_TC_CRDTS(0),
|
||||
MPNIC_RXB_INIT_POOL_TC_CRDTS_P0);
|
||||
mpnic_wr64(mpd, MPNIC_RXB_COMMON_CRDT_CTRL_TC(0),
|
||||
FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_THRESH,
|
||||
MPNIC_RXB_INIT_COMMON_CRDT_THRSH) |
|
||||
FIELD_PREP(MPNIC_RXB_COMMON_CRDT_CTRL_TC_MAX_CRDTS,
|
||||
MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH) |
|
||||
MPNIC_RXB_COMMON_CRDT_CTRL_TC_TC_EN);
|
||||
|
||||
/* Only pool 0 is used, it gets all of the common credits */
|
||||
mpnic_wr64(mpd, MPNIC_RXB_POOL_COMMON_CRDTS(0),
|
||||
MPNIC_RXB_INIT_COMMON_CRDT_MAX_THRSH);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_RXB_BMC_CRDTS(0), MPNIC_RXB_INIT_BMC_CRDTS);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_RXB_MEM_INIT_REQ, MPNIC_MEM_INIT_RXB_VAL);
|
||||
}
|
||||
|
||||
static u64 mpnic_desc_cfg(unsigned int sram_descs, unsigned int q_idx)
|
||||
{
|
||||
/* The hardware encodes the partition size as 64 * 2^n descriptors,
|
||||
* and its start address in units of 64 descriptors.
|
||||
*/
|
||||
return FIELD_PREP(MPNIC_DESC_CFG_NUM_DESCS, __ffs(sram_descs) - 6) |
|
||||
FIELD_PREP(MPNIC_DESC_CFG_START_ADDR,
|
||||
q_idx * (sram_descs >> 6));
|
||||
}
|
||||
|
||||
static void mpnic_desc_sram_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < MPNIC_MAX_TXQS * 2; i++)
|
||||
mpnic_wr64(mpd, MPNIC_TDF_DESC_CFG(i),
|
||||
mpnic_desc_cfg(MPNIC_TDF_SRAM_DESCS, i));
|
||||
|
||||
for (i = 0; i < MPNIC_MAX_RXQS; i++) {
|
||||
mpnic_wr64(mpd, MPNIC_HPQ_DESC_CFG(i),
|
||||
mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS, i));
|
||||
mpnic_wr64(mpd, MPNIC_PPQ_DESC_CFG(i),
|
||||
mpnic_desc_cfg(MPNIC_BDQ_SRAM_DESCS,
|
||||
MPNIC_MAX_RXQS + i));
|
||||
}
|
||||
}
|
||||
|
||||
static void mpnic_rxglb_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
/* Descriptor prefetch reads are only issued once 32 descriptors
|
||||
* worth of FIFO space is available, and no more than 64 descriptors
|
||||
* are fetched for one queue at a time so that a single queue cannot
|
||||
* monopolize the bus. Both have to be multiples of 16 to keep the
|
||||
* reads 128 B aligned.
|
||||
*/
|
||||
mpnic_wr64(mpd, MPNIC_BDQ_GLBL_CTL0,
|
||||
FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_PREFETCH_SPACE_THRESH, 32) |
|
||||
FIELD_PREP(MPNIC_BDQ_GLBL_CTL0_MAX_REQ_SIZE, 64));
|
||||
|
||||
/* Minimum number of descriptors which has to be available before
|
||||
* the descriptor engine considers a queue usable, globally and in
|
||||
* the per-queue prefetch FIFO.
|
||||
*/
|
||||
mpnic_wr64(mpd, MPNIC_RDE_CTL,
|
||||
FIELD_PREP(MPNIC_RDE_CTL_HPQ_DROP_THRESHOLD, 16) |
|
||||
FIELD_PREP(MPNIC_RDE_CTL_PPQ_DROP_THRESHOLD, 16) |
|
||||
FIELD_PREP(MPNIC_RDE_CTL_HPQ_LOCAL_DROP_THRESHOLD, 16) |
|
||||
FIELD_PREP(MPNIC_RDE_CTL_PPQ_LOCAL_DROP_THRESHOLD, 16));
|
||||
|
||||
/* Receive side coalescing is not supported yet */
|
||||
mpnic_wr64(mpd, MPNIC_RSC_GLOBAL_CONF,
|
||||
MPNIC_RSC_GLOBAL_CONF_RSC_DISABLE);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_BDQ_MEM_INIT_REQ, MPNIC_MEM_INIT_BDQ_VAL);
|
||||
mpnic_wr64(mpd, MPNIC_RCM_MEM_INIT_REQ, MPNIC_MEM_INIT_RCM_VAL);
|
||||
mpnic_wr64(mpd, MPNIC_RDE_MEM_INIT_REQ, MPNIC_MEM_INIT_RDE_VAL);
|
||||
mpnic_wr64(mpd, MPNIC_RPC_MEM_INIT_REQ, MPNIC_MEM_INIT_RPC_VAL);
|
||||
}
|
||||
|
||||
static void mpnic_txglb_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
/* Nothing is redirected to the BMC until the Tx offload TCAM gets
|
||||
* programmed with its addresses.
|
||||
*/
|
||||
mpnic_wr64(mpd, MPNIC_TOF_TCAM_DEST_REMAP, 0);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_TCM_MEM_INIT_REQ, MPNIC_MEM_INIT_TCM_VAL);
|
||||
mpnic_wr64(mpd, MPNIC_TDE_MEM_INIT_REQ, MPNIC_MEM_INIT_TDE_VAL);
|
||||
mpnic_wr64(mpd, MPNIC_TDF_MEM_INIT_REQ, MPNIC_MEM_INIT_TDF_VAL);
|
||||
}
|
||||
|
||||
/* Fill the DWRR arbiter memories. Setting the INIT bit makes the hardware
|
||||
* write the given credit and quantum into every entry at the queue, group,
|
||||
* set and port level, so nothing has to be programmed per queue.
|
||||
*/
|
||||
static void mpnic_tqs_sram_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
mpnic_wr64(mpd, MPNIC_TQS_GROUP_INIT_CTL,
|
||||
FIELD_PREP(MPNIC_TQS_GROUP_INIT_CTL_QUANTUM,
|
||||
MPNIC_TQS_DWRR_INIT_QUANTUM) |
|
||||
MPNIC_TQS_GROUP_INIT_CTL_INIT);
|
||||
mpnic_wr64(mpd, MPNIC_TQS_SET_INIT_CTL,
|
||||
FIELD_PREP(MPNIC_TQS_SET_INIT_CTL_QUANTUM,
|
||||
MPNIC_TQS_DWRR_INIT_QUANTUM) |
|
||||
MPNIC_TQS_SET_INIT_CTL_INIT);
|
||||
mpnic_wr64(mpd, MPNIC_TQS_PORT_INIT_CTL,
|
||||
FIELD_PREP(MPNIC_TQS_PORT_INIT_CTL_QUANTUM,
|
||||
MPNIC_TQS_DWRR_INIT_QUANTUM) |
|
||||
MPNIC_TQS_PORT_INIT_CTL_INIT);
|
||||
mpnic_wr64(mpd, MPNIC_TQS_SRAM_INIT_CTL,
|
||||
FIELD_PREP(MPNIC_TQS_SRAM_INIT_CTL_QUANTUM,
|
||||
MPNIC_TQS_DWRR_INIT_QUANTUM) |
|
||||
MPNIC_TQS_SRAM_INIT_CTL_INIT);
|
||||
}
|
||||
|
||||
static void mpnic_tqs_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
u64 val;
|
||||
|
||||
/* Initialize to the largest frame we support, the scheduler
|
||||
* truncates anything above it. The BMC gets the same limit.
|
||||
*/
|
||||
mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL0, MPNIC_TQS_MTU_CTL0_MAX);
|
||||
mpnic_wr64(mpd, MPNIC_TQS_MTU_CTL1, MPNIC_TQS_MTU_CTL0_MAX);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_TQS_GLBL_CTL0,
|
||||
MPNIC_TQS_GLBL_CTL0_TWD_ERROR_CHECK_EN);
|
||||
|
||||
mpnic_tqs_sram_init(mpd);
|
||||
|
||||
/* Only port 0 is used. A single traffic class is in use as well, so
|
||||
* TC0 gets all of the Tx buffer credits not reserved for the BMC.
|
||||
*/
|
||||
mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_0,
|
||||
FIELD_PREP(MPNIC_TQS_GLBL_P0_0_TXB_MAX_CRDTS_0,
|
||||
MPNIC_TQS_GLBL_TXB_CRDT_MAC));
|
||||
mpnic_wr64(mpd, MPNIC_TQS_GLBL_P0_1, 0);
|
||||
mpnic_wr64(mpd, MPNIC_TQS_GLBL_BMC,
|
||||
FIELD_PREP(MPNIC_TQS_GLBL_BMC_TXB_MAX_CRDTS,
|
||||
MPNIC_TQS_GLBL_TXB_CRDT_BMC));
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_TQS_PORT_CTL(0), 0);
|
||||
|
||||
/* Map all sets to port 0 */
|
||||
mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP0(0), ~0ULL);
|
||||
mpnic_wr64(mpd, MPNIC_TQS_SET_P0_MAP1(0), ~0ULL);
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_0,
|
||||
FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_QUEUE,
|
||||
MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) |
|
||||
FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_0_GROUP,
|
||||
MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH));
|
||||
mpnic_wr64(mpd, MPNIC_TQS_CEV_MIN_SCHED_THRESH_1,
|
||||
FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_SET,
|
||||
MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH) |
|
||||
FIELD_PREP(MPNIC_TQS_CEV_MIN_SCHED_THRESH_1_PORT,
|
||||
MPNIC_TQS_DWRR_CEV_MIN_SCHED_THRESH));
|
||||
|
||||
/* The rate limiters are left uninitialized, so shaping has to stay
|
||||
* off or nothing would ever get scheduled.
|
||||
*/
|
||||
mpnic_wr64(mpd, MPNIC_TQS_GLBL_SHAPING, MPNIC_TQS_GLBL_SHAPING_DISABLE);
|
||||
|
||||
/* Enable fairness protection (phantom eligibility). Read modify
|
||||
* write so that the reset default slowdown cycle is preserved.
|
||||
*/
|
||||
val = mpnic_rd64(mpd, MPNIC_TQS_SLOWDOWN_CTL);
|
||||
val |= MPNIC_TQS_SLOWDOWN_CTL_ENABLE;
|
||||
mpnic_wr64(mpd, MPNIC_TQS_SLOWDOWN_CTL, val);
|
||||
|
||||
/* Use immediate credit decrement at every DWRR level so that the
|
||||
* credit reflects a grant in the same cycle.
|
||||
*/
|
||||
val = mpnic_rd64(mpd, MPNIC_TQS_ARB_CTL);
|
||||
val |= MPNIC_TQS_ARB_CTL_SET_CRDT_BUCKET_EN |
|
||||
MPNIC_TQS_ARB_CTL_SET_IMM_DECR_EN |
|
||||
MPNIC_TQS_ARB_CTL_GROUP_CRDT_BUCKET_EN |
|
||||
MPNIC_TQS_ARB_CTL_GROUP_IMM_DECR_EN |
|
||||
MPNIC_TQS_ARB_CTL_QUEUE_CRDT_BUCKET_EN |
|
||||
MPNIC_TQS_ARB_CTL_QUEUE_IMM_DECR_EN;
|
||||
mpnic_wr64(mpd, MPNIC_TQS_ARB_CTL, val);
|
||||
}
|
||||
|
||||
/* The MPS and CLS fields sit at the same bit positions in every block, so
|
||||
* one set of masks covers both the RNI and the TNI registers.
|
||||
*/
|
||||
static void mpnic_mps_init(struct mpnic_dev *mpd, u32 reg, unsigned int mps,
|
||||
unsigned int cls)
|
||||
{
|
||||
u64 val = mpnic_rd64(mpd, reg);
|
||||
|
||||
val &= ~(MPNIC_RNI_RDE_CTL_MPS | MPNIC_RNI_RDE_CTL_CLS);
|
||||
val |= FIELD_PREP(MPNIC_RNI_RDE_CTL_MPS, mps) |
|
||||
FIELD_PREP(MPNIC_RNI_RDE_CTL_CLS, cls);
|
||||
|
||||
mpnic_wr64(mpd, reg, val);
|
||||
}
|
||||
|
||||
/* Likewise for the MRRS and CLS fields, which have their own common
|
||||
* layout.
|
||||
*/
|
||||
static void mpnic_mrrs_init(struct mpnic_dev *mpd, u32 reg, unsigned int mrrs,
|
||||
unsigned int cls)
|
||||
{
|
||||
u64 val = mpnic_rd64(mpd, reg);
|
||||
|
||||
val &= ~(MPNIC_TNI_GLBL_TDF_CTL_MRRS | MPNIC_TNI_GLBL_TDF_CTL_CLS);
|
||||
val |= FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_MRRS, mrrs) |
|
||||
FIELD_PREP(MPNIC_TNI_GLBL_TDF_CTL_CLS, cls);
|
||||
|
||||
mpnic_wr64(mpd, reg, val);
|
||||
}
|
||||
|
||||
/**
|
||||
* mpnic_axi_init - Configure AXI bus parameters from host PCIe capabilities
|
||||
* @mpd: Device to configure
|
||||
*
|
||||
* Programs the max read request size, max payload size and cache line size
|
||||
* of the DMA engines. The hardware encodes all three as a power of 2 index,
|
||||
* MRRS and MPS relative to 128 B and CLS relative to 64 B.
|
||||
*
|
||||
* MAX_OT and MAX_OB are left at their hardware defaults.
|
||||
*/
|
||||
static void mpnic_axi_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
int mps, cls, mrrs;
|
||||
|
||||
mps = clamp(ilog2(mpd->mps) - 7, 0, 3);
|
||||
cls = clamp(ilog2(L1_CACHE_BYTES) - 6, 0, 3);
|
||||
|
||||
mpnic_mps_init(mpd, MPNIC_RNI_RDE_CTL, mps, cls);
|
||||
mpnic_mps_init(mpd, MPNIC_RNI_RCM_CTL, mps, cls);
|
||||
mpnic_mps_init(mpd, MPNIC_TNI_GLBL_TCM_CTL, mps, cls);
|
||||
|
||||
mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 3);
|
||||
mpnic_mrrs_init(mpd, MPNIC_RNI_RBP_CTL, mrrs, cls);
|
||||
mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDF_CTL, mrrs, cls);
|
||||
|
||||
/* TDE supports a wider range of MRRS encodings. */
|
||||
mrrs = clamp(ilog2(mpd->readrq) - 7, 0, 5);
|
||||
mpnic_mrrs_init(mpd, MPNIC_TNI_GLBL_TDE_CTL, mrrs, cls);
|
||||
}
|
||||
|
||||
/**
|
||||
* mpnic_ro_init - Set relaxed ordering on the outbound TLP attributes
|
||||
* @mpd: Device to configure
|
||||
*
|
||||
* Completions must stay ordered so that they are not observed before the
|
||||
* payload DMA they describe has landed, so RCM and TCM are left alone.
|
||||
*/
|
||||
static void mpnic_ro_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
u64 attr = mpd->relaxed_ord ? MPNIC_OB_ATTR_RO : 0;
|
||||
|
||||
mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_H, attr);
|
||||
mpnic_wr64(mpd, MPNIC_OB_ATTR_TDE_P, attr);
|
||||
mpnic_wr64(mpd, MPNIC_OB_ATTR_TDF, attr);
|
||||
mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_HPQ, attr);
|
||||
mpnic_wr64(mpd, MPNIC_OB_ATTR_RBP_PPQ, attr);
|
||||
mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_H, attr);
|
||||
mpnic_wr64(mpd, MPNIC_OB_ATTR_RDE_P, attr);
|
||||
}
|
||||
|
||||
static bool mpnic_init_status_ready(struct mpnic_dev *mpd,
|
||||
const struct mpnic_init_poll *polls,
|
||||
struct mpnic_poll_state *state)
|
||||
{
|
||||
u64 val;
|
||||
int i;
|
||||
|
||||
for (i = state->poll_idx; polls[i].addr; i++) {
|
||||
val = mpnic_rd64(mpd, polls[i].addr);
|
||||
|
||||
if ((val & polls[i].exp_val) != polls[i].exp_val) {
|
||||
state->poll_idx = i;
|
||||
state->val = val;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* mpnic_mem_init_poll - Wait for the memory initializations to complete
|
||||
* @mpd: Device to poll
|
||||
*
|
||||
* The blocks initialize their memories in parallel, so walk the status
|
||||
* registers in order and only go back to sleep on the first one which is
|
||||
* not done yet.
|
||||
*
|
||||
* Return: 0 on success, -ETIMEDOUT if not everything completed in time
|
||||
*/
|
||||
static int mpnic_mem_init_poll(struct mpnic_dev *mpd)
|
||||
{
|
||||
static const struct mpnic_init_poll polls[] = {
|
||||
{ MPNIC_TQS_ARB_INIT_VAL, MPNIC_TQS_SRAM_STS },
|
||||
{ MPNIC_MEM_INIT_BDQ_VAL, MPNIC_BDQ_MEM_INIT_DONE },
|
||||
{ MPNIC_MEM_INIT_RCM_VAL, MPNIC_RCM_MEM_INIT_DONE },
|
||||
{ MPNIC_MEM_INIT_RDE_VAL, MPNIC_RDE_MEM_INIT_DONE },
|
||||
{ MPNIC_MEM_INIT_RPC_VAL, MPNIC_RPC_MEM_INIT_DONE },
|
||||
{ MPNIC_MEM_INIT_TCM_VAL, MPNIC_TCM_MEM_INIT_DONE },
|
||||
{ MPNIC_MEM_INIT_TDE_VAL, MPNIC_TDE_MEM_INIT_DONE },
|
||||
{ MPNIC_MEM_INIT_TDF_VAL, MPNIC_TDF_MEM_INIT_DONE },
|
||||
{ MPNIC_MEM_INIT_RXB_VAL, MPNIC_RXB_MEM_INIT_DONE },
|
||||
{ 0 },
|
||||
};
|
||||
struct mpnic_poll_state state = {};
|
||||
bool done;
|
||||
int err;
|
||||
|
||||
err = read_poll_timeout(mpnic_init_status_ready, done, done,
|
||||
MPNIC_MEM_INIT_POLL_US, MPNIC_MEM_INIT_TO_US,
|
||||
false, mpd, polls, &state);
|
||||
if (err)
|
||||
dev_err(mpd->dev, "Poll timeout for reg 0x%x: 0x%llx\n",
|
||||
polls[state.poll_idx].addr, state.val);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
int mpnic_dev_init(struct mpnic_dev *mpd)
|
||||
{
|
||||
int err;
|
||||
|
||||
mpnic_tdf_glbl_init(mpd);
|
||||
mpnic_txb_init(mpd);
|
||||
mpnic_rxb_init(mpd);
|
||||
mpnic_desc_sram_init(mpd);
|
||||
mpnic_axi_init(mpd);
|
||||
mpnic_ro_init(mpd);
|
||||
mpnic_rxglb_init(mpd);
|
||||
mpnic_txglb_init(mpd);
|
||||
mpnic_tqs_init(mpd);
|
||||
|
||||
err = mpnic_mem_init_poll(mpd);
|
||||
if (err) {
|
||||
dev_err(mpd->dev, "Device initialization failed: %d\n", err);
|
||||
return err;
|
||||
}
|
||||
|
||||
if (!mpnic_present(mpd))
|
||||
return -EIO;
|
||||
|
||||
return 0;
|
||||
}
|
||||
64
drivers/net/ethernet/meta/mpnic/mpnic_irq.c
Normal file
64
drivers/net/ethernet/meta/mpnic/mpnic_irq.c
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#include <linux/cpumask.h>
|
||||
#include <linux/interrupt.h>
|
||||
#include <linux/minmax.h>
|
||||
#include <linux/pci.h>
|
||||
|
||||
#include "mpnic.h"
|
||||
|
||||
int mpnic_request_irq(struct mpnic_dev *mpd, int nr, irq_handler_t handler,
|
||||
unsigned long flags, const char *name, void *data)
|
||||
{
|
||||
struct pci_dev *pdev = to_pci_dev(mpd->dev);
|
||||
int irq = pci_irq_vector(pdev, nr);
|
||||
|
||||
if (irq < 0)
|
||||
return irq;
|
||||
|
||||
return request_irq(irq, handler, flags, name, data);
|
||||
}
|
||||
|
||||
void mpnic_free_irq(struct mpnic_dev *mpd, int nr, void *data)
|
||||
{
|
||||
struct pci_dev *pdev = to_pci_dev(mpd->dev);
|
||||
int irq = pci_irq_vector(pdev, nr);
|
||||
|
||||
if (irq < 0)
|
||||
return;
|
||||
|
||||
free_irq(irq, data);
|
||||
}
|
||||
|
||||
void mpnic_free_irqs(struct mpnic_dev *mpd)
|
||||
{
|
||||
struct pci_dev *pdev = to_pci_dev(mpd->dev);
|
||||
|
||||
mpd->num_irqs = 0;
|
||||
pci_free_irq_vectors(pdev);
|
||||
}
|
||||
|
||||
int mpnic_alloc_irqs(struct mpnic_dev *mpd)
|
||||
{
|
||||
unsigned int wanted_irqs = MPNIC_NON_NAPI_VECTORS;
|
||||
struct pci_dev *pdev = to_pci_dev(mpd->dev);
|
||||
int num_irqs;
|
||||
|
||||
wanted_irqs += min_t(unsigned int, num_online_cpus(), MPNIC_MAX_RXQS);
|
||||
num_irqs = pci_alloc_irq_vectors(pdev, MPNIC_NON_NAPI_VECTORS + 1,
|
||||
wanted_irqs, PCI_IRQ_MSIX);
|
||||
if (num_irqs < 0) {
|
||||
dev_err(mpd->dev, "Failed to allocate MSI-X entries: %d\n",
|
||||
num_irqs);
|
||||
return num_irqs;
|
||||
}
|
||||
|
||||
if (num_irqs < wanted_irqs)
|
||||
dev_warn(mpd->dev, "Allocated %d IRQs, expected %u\n",
|
||||
num_irqs, wanted_irqs);
|
||||
|
||||
mpd->num_irqs = num_irqs;
|
||||
|
||||
return 0;
|
||||
}
|
||||
186
drivers/net/ethernet/meta/mpnic/mpnic_netdev.c
Normal file
186
drivers/net/ethernet/meta/mpnic/mpnic_netdev.c
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#include <linux/etherdevice.h>
|
||||
#include <linux/ipv6.h>
|
||||
#include <linux/netdevice.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/types.h>
|
||||
|
||||
#include "mpnic.h"
|
||||
#include "mpnic_netdev.h"
|
||||
#include "mpnic_txrx.h"
|
||||
|
||||
static int mpnic_open(struct net_device *netdev)
|
||||
{
|
||||
struct mpnic_net *mpn = netdev_priv(netdev);
|
||||
int err;
|
||||
|
||||
err = mpnic_alloc_napi_vectors(mpn);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
err = mpnic_alloc_resources(mpn);
|
||||
if (err)
|
||||
goto err_free_napi_vectors;
|
||||
|
||||
err = mpnic_set_netif_queues(mpn);
|
||||
if (err)
|
||||
goto err_free_resources;
|
||||
|
||||
mpnic_enable(mpn);
|
||||
mpnic_fill(mpn);
|
||||
mpnic_napi_enable(mpn);
|
||||
|
||||
netif_tx_wake_all_queues(netdev);
|
||||
netif_carrier_on(netdev);
|
||||
|
||||
return 0;
|
||||
|
||||
err_free_resources:
|
||||
mpnic_free_resources(mpn);
|
||||
err_free_napi_vectors:
|
||||
mpnic_free_napi_vectors(mpn);
|
||||
return err;
|
||||
}
|
||||
|
||||
static int mpnic_stop(struct net_device *netdev)
|
||||
{
|
||||
struct mpnic_net *mpn = netdev_priv(netdev);
|
||||
|
||||
netif_carrier_off(netdev);
|
||||
|
||||
mpnic_napi_disable(mpn);
|
||||
netif_tx_disable(netdev);
|
||||
|
||||
mpnic_disable(mpn);
|
||||
mpnic_wait_all_queues_idle(mpn->mpd);
|
||||
mpnic_flush(mpn);
|
||||
|
||||
mpnic_reset_netif_queues(mpn);
|
||||
mpnic_free_resources(mpn);
|
||||
mpnic_free_napi_vectors(mpn);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct net_device_ops mpnic_netdev_ops = {
|
||||
.ndo_open = mpnic_open,
|
||||
.ndo_stop = mpnic_stop,
|
||||
.ndo_validate_addr = eth_validate_addr,
|
||||
.ndo_start_xmit = mpnic_xmit_frame,
|
||||
};
|
||||
|
||||
/**
|
||||
* mpnic_netdev_free - Free the netdev associated with mpnic
|
||||
* @mpd: Driver specific structure to free netdev from
|
||||
**/
|
||||
void mpnic_netdev_free(struct mpnic_dev *mpd)
|
||||
{
|
||||
free_netdev(mpd->netdev);
|
||||
mpd->netdev = NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* mpnic_netdev_alloc - Allocate a netdev and associate it with mpnic
|
||||
* @mpd: Driver specific structure to associate the netdev with
|
||||
*
|
||||
* Return: NULL on failure.
|
||||
**/
|
||||
struct net_device *mpnic_netdev_alloc(struct mpnic_dev *mpd)
|
||||
{
|
||||
struct net_device *netdev;
|
||||
struct mpnic_net *mpn;
|
||||
unsigned int queues;
|
||||
|
||||
netdev = alloc_etherdev_mq(sizeof(*mpn), MPNIC_MAX_RXQS);
|
||||
if (!netdev)
|
||||
return NULL;
|
||||
|
||||
SET_NETDEV_DEV(netdev, mpd->dev);
|
||||
mpd->netdev = netdev;
|
||||
|
||||
netdev->netdev_ops = &mpnic_netdev_ops;
|
||||
netdev->request_ops_lock = true;
|
||||
|
||||
mpn = netdev_priv(netdev);
|
||||
mpn->netdev = netdev;
|
||||
mpn->mpd = mpd;
|
||||
|
||||
mpn->txq_size = MPNIC_TXQ_SIZE_DEFAULT;
|
||||
mpn->hpq_size = MPNIC_HPQ_SIZE_DEFAULT;
|
||||
mpn->ppq_size = MPNIC_PPQ_SIZE_DEFAULT;
|
||||
mpn->rcq_size = MPNIC_RCQ_SIZE_DEFAULT;
|
||||
|
||||
queues = min(netif_get_num_default_rss_queues(),
|
||||
mpd->num_irqs - MPNIC_NON_NAPI_VECTORS);
|
||||
mpn->num_tx_queues = queues;
|
||||
mpn->num_rx_queues = queues;
|
||||
mpn->num_napi = queues;
|
||||
|
||||
netdev->features |= NETIF_F_SG;
|
||||
netdev->hw_features |= netdev->features;
|
||||
netdev->vlan_features |= netdev->features;
|
||||
|
||||
netdev->min_mtu = IPV6_MIN_MTU;
|
||||
netdev->max_mtu = MPNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN;
|
||||
|
||||
netif_carrier_off(netdev);
|
||||
netif_tx_stop_all_queues(netdev);
|
||||
|
||||
return netdev;
|
||||
}
|
||||
|
||||
static int mpnic_dsn_to_mac_addr(u64 dsn, u8 *addr)
|
||||
{
|
||||
addr[0] = (dsn >> 56) & 0xFF;
|
||||
addr[1] = (dsn >> 48) & 0xFF;
|
||||
addr[2] = (dsn >> 40) & 0xFF;
|
||||
addr[3] = (dsn >> 16) & 0xFF;
|
||||
addr[4] = (dsn >> 8) & 0xFF;
|
||||
addr[5] = dsn & 0xFF;
|
||||
|
||||
return is_valid_ether_addr(addr) ? 0 : -EINVAL;
|
||||
}
|
||||
|
||||
/**
|
||||
* mpnic_netdev_register - Assign the MAC address and register the netdev
|
||||
* @netdev: Netdev to register
|
||||
*
|
||||
* The permanent address is derived from the PCIe device serial number, the
|
||||
* same way the firmware and the BMC derive it. A random address would break
|
||||
* provisioning, so refuse to spawn the interface if the serial number does
|
||||
* not yield a valid one.
|
||||
*
|
||||
* Return: non-zero on failure.
|
||||
**/
|
||||
int mpnic_netdev_register(struct net_device *netdev)
|
||||
{
|
||||
struct mpnic_net *mpn = netdev_priv(netdev);
|
||||
struct mpnic_dev *mpd = mpn->mpd;
|
||||
u8 addr[ETH_ALEN];
|
||||
int err;
|
||||
|
||||
err = mpnic_dsn_to_mac_addr(mpd->dsn, addr);
|
||||
if (err) {
|
||||
dev_err(mpd->dev, "MAC addr %pM invalid\n", addr);
|
||||
return err;
|
||||
}
|
||||
|
||||
ether_addr_copy(netdev->perm_addr, addr);
|
||||
eth_hw_addr_set(netdev, addr);
|
||||
|
||||
err = netif_set_real_num_queues(netdev, mpn->num_tx_queues,
|
||||
mpn->num_rx_queues);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
/* Abort if MMIO has failed. This has to be the last check before
|
||||
* registration, the register accessors can only detach the device
|
||||
* once it has been registered.
|
||||
*/
|
||||
if (!mpnic_present(mpd))
|
||||
return -EIO;
|
||||
|
||||
return register_netdev(netdev);
|
||||
}
|
||||
35
drivers/net/ethernet/meta/mpnic/mpnic_netdev.h
Normal file
35
drivers/net/ethernet/meta/mpnic/mpnic_netdev.h
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#ifndef _MPNIC_NETDEV_H_
|
||||
#define _MPNIC_NETDEV_H_
|
||||
|
||||
#include <linux/types.h>
|
||||
|
||||
#include "mpnic.h"
|
||||
#include "mpnic_txrx.h"
|
||||
|
||||
struct mpnic_net {
|
||||
struct mpnic_ring *tx[MPNIC_MAX_TXQS];
|
||||
struct mpnic_ring *rx[MPNIC_MAX_RXQS];
|
||||
|
||||
struct mpnic_napi_vector *napi[MPNIC_MAX_NAPI_VECTORS];
|
||||
|
||||
struct net_device *netdev;
|
||||
struct mpnic_dev *mpd;
|
||||
|
||||
u32 txq_size;
|
||||
u32 hpq_size;
|
||||
u32 ppq_size;
|
||||
u32 rcq_size;
|
||||
|
||||
u16 num_napi;
|
||||
u16 num_tx_queues;
|
||||
u16 num_rx_queues;
|
||||
};
|
||||
|
||||
struct net_device *mpnic_netdev_alloc(struct mpnic_dev *mpd);
|
||||
void mpnic_netdev_free(struct mpnic_dev *mpd);
|
||||
int mpnic_netdev_register(struct net_device *netdev);
|
||||
|
||||
#endif /* _MPNIC_NETDEV_H_ */
|
||||
187
drivers/net/ethernet/meta/mpnic/mpnic_pci.c
Normal file
187
drivers/net/ethernet/meta/mpnic/mpnic_pci.c
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#include <linux/dma-mapping.h>
|
||||
#include <linux/err.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/netdevice.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/types.h>
|
||||
|
||||
#include "mpnic.h"
|
||||
#include "mpnic_netdev.h"
|
||||
|
||||
#define PCI_DEVICE_ID_META_MPNIC 0x0014
|
||||
|
||||
static void mpnic_mmio_err(struct mpnic_dev *mpd, u32 reg)
|
||||
{
|
||||
/* Hardware is giving us all 1's reads, assume it is gone */
|
||||
WRITE_ONCE(mpd->uc_addr0, NULL);
|
||||
|
||||
dev_err(mpd->dev,
|
||||
"Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n",
|
||||
reg, reg << 2);
|
||||
|
||||
/* Tell the stack the device has lost its PCIe link */
|
||||
if (mpd->netdev)
|
||||
netif_device_detach(mpd->netdev);
|
||||
}
|
||||
|
||||
u64 mpnic_rd64(struct mpnic_dev *mpd, u32 reg)
|
||||
{
|
||||
u32 __iomem *csr = READ_ONCE(mpd->uc_addr0);
|
||||
u64 value;
|
||||
|
||||
if (!csr)
|
||||
return ~0ULL;
|
||||
|
||||
value = readq(csr + reg);
|
||||
|
||||
/* If any bits are 0 value should be valid */
|
||||
if (~value)
|
||||
return value;
|
||||
|
||||
/* All ones can be a valid value, so confirm against a register
|
||||
* which never reads that way on a live device.
|
||||
*/
|
||||
if (reg != MPNIC_BDQ_SPARE && ~readq(csr + MPNIC_BDQ_SPARE))
|
||||
return value;
|
||||
|
||||
mpnic_mmio_err(mpd, reg);
|
||||
|
||||
return ~0ULL;
|
||||
}
|
||||
|
||||
static struct mpnic_dev *mpnic_alloc(struct pci_dev *pdev)
|
||||
{
|
||||
struct mpnic_dev *mpd;
|
||||
|
||||
mpd = kzalloc_obj(*mpd);
|
||||
if (!mpd)
|
||||
return NULL;
|
||||
|
||||
pci_set_drvdata(pdev, mpd);
|
||||
mpd->dev = &pdev->dev;
|
||||
|
||||
mpd->dsn = pci_get_dsn(pdev);
|
||||
mpd->mps = pcie_get_mps(pdev);
|
||||
mpd->readrq = pcie_get_readrq(pdev);
|
||||
mpd->relaxed_ord = pcie_relaxed_ordering_enabled(pdev);
|
||||
|
||||
return mpd;
|
||||
}
|
||||
|
||||
/**
|
||||
* mpnic_probe - Device initialization routine
|
||||
* @pdev: PCI device information struct
|
||||
* @ent: entry in mpnic_pci_tbl
|
||||
*
|
||||
* Return: 0 on success, negative on failure
|
||||
**/
|
||||
static int mpnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
||||
{
|
||||
struct net_device *netdev;
|
||||
void __iomem *uc_addr0;
|
||||
struct mpnic_dev *mpd;
|
||||
int err;
|
||||
|
||||
if (pdev->error_state != pci_channel_io_normal) {
|
||||
dev_err(&pdev->dev,
|
||||
"PCI device still in an error state. Unable to load...\n");
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
err = pcim_enable_device(pdev);
|
||||
if (err) {
|
||||
dev_err(&pdev->dev, "PCI enable device failed: %d\n", err);
|
||||
return err;
|
||||
}
|
||||
|
||||
err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46));
|
||||
if (err) {
|
||||
dev_err(&pdev->dev, "DMA configuration failed: %d\n", err);
|
||||
return err;
|
||||
}
|
||||
|
||||
mpd = mpnic_alloc(pdev);
|
||||
if (!mpd)
|
||||
return -ENOMEM;
|
||||
|
||||
uc_addr0 = pcim_iomap_region(pdev, 0, MPNIC_DRV_NAME);
|
||||
if (IS_ERR(uc_addr0)) {
|
||||
err = PTR_ERR(uc_addr0);
|
||||
dev_err(&pdev->dev, "Mapping the register file failed: %d\n",
|
||||
err);
|
||||
goto err_free_mpd;
|
||||
}
|
||||
mpd->uc_addr0 = uc_addr0;
|
||||
|
||||
pci_set_master(pdev);
|
||||
pci_save_state(pdev);
|
||||
|
||||
err = mpnic_alloc_irqs(mpd);
|
||||
if (err)
|
||||
goto err_free_mpd;
|
||||
|
||||
err = mpnic_dev_init(mpd);
|
||||
if (err)
|
||||
goto err_free_irqs;
|
||||
|
||||
netdev = mpnic_netdev_alloc(mpd);
|
||||
if (!netdev) {
|
||||
dev_err(&pdev->dev, "Netdev allocation failed\n");
|
||||
err = -ENOMEM;
|
||||
goto err_free_irqs;
|
||||
}
|
||||
|
||||
err = mpnic_netdev_register(netdev);
|
||||
if (err) {
|
||||
dev_err(&pdev->dev, "Netdev registration failed: %d\n", err);
|
||||
goto err_free_netdev;
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
||||
err_free_netdev:
|
||||
mpnic_netdev_free(mpd);
|
||||
err_free_irqs:
|
||||
mpnic_free_irqs(mpd);
|
||||
err_free_mpd:
|
||||
kfree(mpd);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
/**
|
||||
* mpnic_remove - Device removal routine
|
||||
* @pdev: PCI device information struct
|
||||
**/
|
||||
static void mpnic_remove(struct pci_dev *pdev)
|
||||
{
|
||||
struct mpnic_dev *mpd = pci_get_drvdata(pdev);
|
||||
|
||||
unregister_netdev(mpd->netdev);
|
||||
mpnic_netdev_free(mpd);
|
||||
mpnic_free_irqs(mpd);
|
||||
kfree(mpd);
|
||||
}
|
||||
|
||||
static const struct pci_device_id mpnic_pci_tbl[] = {
|
||||
{ PCI_VDEVICE(META, PCI_DEVICE_ID_META_MPNIC) },
|
||||
/* required last entry */
|
||||
{}
|
||||
};
|
||||
MODULE_DEVICE_TABLE(pci, mpnic_pci_tbl);
|
||||
|
||||
static struct pci_driver mpnic_driver = {
|
||||
.name = MPNIC_DRV_NAME,
|
||||
.id_table = mpnic_pci_tbl,
|
||||
.probe = mpnic_probe,
|
||||
.remove = mpnic_remove,
|
||||
};
|
||||
|
||||
module_pci_driver(mpnic_driver);
|
||||
|
||||
MODULE_DESCRIPTION("Meta Platforms Network Interface Controller");
|
||||
MODULE_LICENSE("GPL");
|
||||
1395
drivers/net/ethernet/meta/mpnic/mpnic_txrx.c
Normal file
1395
drivers/net/ethernet/meta/mpnic/mpnic_txrx.c
Normal file
File diff suppressed because it is too large
Load Diff
151
drivers/net/ethernet/meta/mpnic/mpnic_txrx.h
Normal file
151
drivers/net/ethernet/meta/mpnic/mpnic_txrx.h
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/* Copyright (c) Meta Platforms, Inc. and affiliates. */
|
||||
|
||||
#ifndef _MPNIC_TXRX_H_
|
||||
#define _MPNIC_TXRX_H_
|
||||
|
||||
#include <linux/if_ether.h>
|
||||
#include <linux/netdevice.h>
|
||||
#include <linux/skbuff.h>
|
||||
#include <linux/types.h>
|
||||
#include <net/netdev_queues.h>
|
||||
#include <net/xdp.h>
|
||||
|
||||
#include "mpnic.h"
|
||||
|
||||
struct mpnic_net;
|
||||
|
||||
/* Space we have to have available in a work queue to take a packet:
|
||||
* 1 descriptor per page
|
||||
* + 1 descriptor for the skb head
|
||||
* + 1 descriptor for the metadata
|
||||
* + 7 descriptors to keep the tail out of the head's cacheline
|
||||
* If we cannot guarantee that we return NETDEV_TX_BUSY.
|
||||
*/
|
||||
#define MPNIC_MAX_SKB_DESC (MAX_SKB_FRAGS + 9)
|
||||
#define MPNIC_TX_DESC_WAKEUP (MPNIC_MAX_SKB_DESC * 2)
|
||||
|
||||
#define MPNIC_MAX_NAPI_VECTORS 1024u
|
||||
|
||||
/* Number of buffer descriptors the driver posts before ringing the
|
||||
* doorbell. The device consumes whatever the doorbell points at, this is
|
||||
* purely to keep the driver from writing the CSR for every descriptor.
|
||||
*/
|
||||
#define MPNIC_BDQ_BATCH_SIZE 64u
|
||||
|
||||
#define MPNIC_TXQ_SIZE_DEFAULT 1024
|
||||
#define MPNIC_HPQ_SIZE_DEFAULT 256
|
||||
#define MPNIC_PPQ_SIZE_DEFAULT 256
|
||||
#define MPNIC_RCQ_SIZE_DEFAULT 1024
|
||||
|
||||
/* Room the device has to leave in front of and behind every header so the
|
||||
* driver can build an skb around it in place. The headroom is padded out
|
||||
* so that consecutive headers in one page start 128 B aligned.
|
||||
*/
|
||||
#define MPNIC_RX_TROOM \
|
||||
SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
|
||||
#define MPNIC_RX_HROOM \
|
||||
(ALIGN(MPNIC_RX_TROOM + XDP_PACKET_HEADROOM, 128) - MPNIC_RX_TROOM)
|
||||
|
||||
/* Headers longer than this are split off into the payload queue */
|
||||
#define MPNIC_RX_MAX_HDR 1536
|
||||
|
||||
/* A page is handed out to many packets, each of which takes one reference.
|
||||
* Rather than a locked increment per packet the driver takes a batch of
|
||||
* references up front and returns whatever is left when the page is done.
|
||||
*/
|
||||
#define MPNIC_PAGECNT_BIAS_MAX (PAGE_SIZE + 1)
|
||||
|
||||
#define MPNIC_MAX_JUMBO_FRAME_SIZE 9742
|
||||
|
||||
/* The page a buffer descriptor queue is currently handing out. Records
|
||||
* how many of the references taken on it are still unused.
|
||||
*/
|
||||
struct mpnic_pg_ctxt {
|
||||
struct page *page;
|
||||
long pagecnt_bias;
|
||||
u32 idx;
|
||||
};
|
||||
|
||||
struct mpnic_pkt_ctxt {
|
||||
struct xdp_buff buff;
|
||||
bool add_frag_failed;
|
||||
};
|
||||
|
||||
struct mpnic_rcq_state {
|
||||
struct mpnic_pkt_ctxt pkt;
|
||||
struct mpnic_pg_ctxt hdr;
|
||||
struct mpnic_pg_ctxt payld;
|
||||
};
|
||||
|
||||
struct mpnic_ring {
|
||||
union {
|
||||
struct mpnic_rcq_state *state; /* RCQ */
|
||||
struct page **rx_buf; /* BDQ */
|
||||
void **tx_buf; /* TWQ */
|
||||
void *buffer; /* Generic pointer */
|
||||
};
|
||||
|
||||
u32 __iomem *doorbell; /* Pointer to CSR space for ring */
|
||||
__le64 *desc; /* Descriptor ring memory */
|
||||
u16 size_mask; /* Size of ring in descriptors - 1 */
|
||||
u16 q_idx; /* Hardware queue index */
|
||||
|
||||
u32 head, tail; /* Head/Tail of ring */
|
||||
|
||||
union {
|
||||
/* BDQ only */
|
||||
struct page_pool *page_pool;
|
||||
|
||||
/* TWQ only, index of the metadata descriptor of the last
|
||||
* packet placed in the ring without ringing the doorbell,
|
||||
* -1 if the doorbell is in sync with the tail.
|
||||
*/
|
||||
s32 deferred_meta;
|
||||
};
|
||||
|
||||
/* Slow path fields follow */
|
||||
dma_addr_t dma; /* Phys addr of descriptor memory */
|
||||
size_t size; /* Size of descriptor ring in memory */
|
||||
};
|
||||
|
||||
/* The device pairs two work queues with one completion queue. On the Rx
|
||||
* side they are the header and the payload buffer descriptor queues; on
|
||||
* the Tx side only the first one is used for now, the second one becomes
|
||||
* the XDP ring.
|
||||
*/
|
||||
struct mpnic_q_triad {
|
||||
struct xdp_rxq_info xdp_rxq;
|
||||
struct mpnic_ring sub0, sub1, cmpl;
|
||||
};
|
||||
|
||||
struct mpnic_napi_vector {
|
||||
struct napi_struct napi;
|
||||
struct device *dev; /* Device for DMA unmapping */
|
||||
struct mpnic_dev *mpd;
|
||||
|
||||
u16 v_idx;
|
||||
u16 txt_count;
|
||||
u16 rxt_count;
|
||||
|
||||
char name[IFNAMSIZ + 11];
|
||||
|
||||
struct mpnic_q_triad qt[];
|
||||
};
|
||||
|
||||
netdev_tx_t mpnic_xmit_frame(struct sk_buff *skb, struct net_device *dev);
|
||||
int mpnic_alloc_napi_vectors(struct mpnic_net *mpn);
|
||||
void mpnic_free_napi_vectors(struct mpnic_net *mpn);
|
||||
int mpnic_alloc_resources(struct mpnic_net *mpn);
|
||||
void mpnic_free_resources(struct mpnic_net *mpn);
|
||||
int mpnic_set_netif_queues(struct mpnic_net *mpn);
|
||||
void mpnic_reset_netif_queues(struct mpnic_net *mpn);
|
||||
void mpnic_napi_enable(struct mpnic_net *mpn);
|
||||
void mpnic_napi_disable(struct mpnic_net *mpn);
|
||||
void mpnic_enable(struct mpnic_net *mpn);
|
||||
void mpnic_disable(struct mpnic_net *mpn);
|
||||
void mpnic_wait_all_queues_idle(struct mpnic_dev *mpd);
|
||||
void mpnic_flush(struct mpnic_net *mpn);
|
||||
void mpnic_fill(struct mpnic_net *mpn);
|
||||
|
||||
#endif /* _MPNIC_TXRX_H_ */
|
||||
Loading…
Reference in New Issue
Block a user