diff --git a/drivers/net/ethernet/mucse/Kconfig b/drivers/net/ethernet/mucse/Kconfig index 0b3e853d625f..c324118dae2b 100644 --- a/drivers/net/ethernet/mucse/Kconfig +++ b/drivers/net/ethernet/mucse/Kconfig @@ -19,6 +19,8 @@ if NET_VENDOR_MUCSE config MGBE tristate "Mucse(R) 1GbE PCI Express adapters support" depends on PCI + depends on PCI_MSI + select PAGE_POOL help This driver supports Mucse(R) 1GbE PCI Express family of adapters. diff --git a/drivers/net/ethernet/mucse/rnpgbe/Makefile b/drivers/net/ethernet/mucse/rnpgbe/Makefile index de8bcb7772ab..17574cad392a 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/Makefile +++ b/drivers/net/ethernet/mucse/rnpgbe/Makefile @@ -8,4 +8,5 @@ obj-$(CONFIG_MGBE) += rnpgbe.o rnpgbe-objs := rnpgbe_main.o\ rnpgbe_chip.o\ rnpgbe_mbx.o\ - rnpgbe_mbx_fw.o + rnpgbe_mbx_fw.o\ + rnpgbe_lib.o diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe.h b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe.h index 5b024f9f7e17..4e9776b28bfb 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe.h +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe.h @@ -5,7 +5,14 @@ #define _RNPGBE_H #include +#include #include +#include +#include +#include +#include + +#include "rnpgbe_hw.h" enum rnpgbe_boards { board_n500, @@ -26,38 +33,229 @@ struct mucse_mbx_info { u32 fwpf_ctrl_base; }; -/* Enum for firmware notification modes, - * more modes (e.g., portup, link_report) will be added in future - **/ enum { mucse_fw_powerup, + mucse_fw_portup, + mucse_fw_link_report_en, }; struct mucse_hw { void __iomem *hw_addr; + void __iomem *ring_msix_base; struct pci_dev *pdev; struct mucse_mbx_info mbx; int port; + int speed; + bool link; + u16 cycles_per_us; u8 pfvfnum; + u8 duplex; }; -struct mucse_stats { - u64 tx_dropped; +struct rnpgbe_tx_desc { + __le64 pkt_addr; /* Packet buffer address */ + union { + __le64 vlan_cmd_bsz; + struct { + __le32 blen_mac_ip_len; + __le32 vlan_cmd; /* vlan & cmd status */ + }; + }; +#define M_TXD_CMD_RS 0x040000 /* Report Status */ +#define M_TXD_STAT_DD 0x020000 /* Descriptor Done */ +#define M_TXD_CMD_EOP 0x010000 /* End of Packet */ +}; + +union rnpgbe_rx_desc { + struct { + __le64 pkt_addr; /* Packet buffer address */ + __le64 resv_cmd; /* cmd status */ + }; + struct { + __le32 rss_hash; /* RSS HASH */ + __le16 mark; /* mark info */ + __le16 rev1; + __le16 len; /* Packet length */ + __le16 padding_len; + __le16 vlan; /* VLAN tag */ + __le16 cmd; /* cmd status */ +#define M_RXD_STAT_DD BIT(1) /* Descriptor Done */ +#define M_RXD_STAT_EOP BIT(0) /* End of Packet */ + } wb; +}; + +#define M_TX_DESC(R, i) (&(((struct rnpgbe_tx_desc *)((R)->desc))[i])) +#define M_RX_DESC(R, i) (&(((union rnpgbe_rx_desc *)((R)->desc))[i])) + +static inline __le16 rnpgbe_test_staterr(union rnpgbe_rx_desc *rx_desc, + const u16 stat_err_bits) +{ + return rx_desc->wb.cmd & cpu_to_le16(stat_err_bits); +} + +struct mucse_tx_buffer { + struct rnpgbe_tx_desc *next_to_watch; + struct sk_buff *skb; + unsigned int bytecount; + unsigned short gso_segs; + DEFINE_DMA_UNMAP_ADDR(dma); + DEFINE_DMA_UNMAP_LEN(len); + bool mapped_as_page; /* true if dma was mapped with dma_map_page */ +}; + +struct mucse_queue_stats { + u64 packets; + u64 bytes; + u64 length_errors; + atomic64_t dropped; + struct u64_stats_sync syncp; +} ____cacheline_aligned_in_smp; + +struct mucse_rx_buffer { + struct sk_buff *skb; + dma_addr_t dma; + struct page *page; + u32 page_offset; +}; + +enum mucse_rx_drop_reason { + MUCSE_RX_DROP_NONE, + MUCSE_RX_DROP_RESOURCE, + MUCSE_RX_DROP_LENGTH, +}; + +struct mucse_ring { + struct mucse_ring *next; + struct mucse_q_vector *q_vector; + struct net_device *netdev; + struct device *dev; + struct page_pool *page_pool; + void *desc; + union { + struct mucse_tx_buffer *tx_buffer_info; + struct mucse_rx_buffer *rx_buffer_info; + }; + void __iomem *ring_addr; + void __iomem *tail; + void __iomem *irq_mask; + void __iomem *trig; + u8 queue_index; + /* hw ring idx */ + u8 rnpgbe_queue_idx; + u8 pfvfnum; + u16 count; + u16 next_to_use; + u16 next_to_clean; + dma_addr_t dma; + unsigned int size; + u32 drop_bytes; /* bytes in the current frame being dropped */ + struct mucse_queue_stats *stats; + enum mucse_rx_drop_reason drop_reason; +} ____cacheline_internodealigned_in_smp; + +/* Refill RX descriptors in batches of this size. */ +#define M_RX_BUFFER_WRITE 16 + +static inline u16 mucse_desc_unused(struct mucse_ring *ring) +{ + u16 ntc = ring->next_to_clean; + u16 ntu = ring->next_to_use; + + return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1; +} + +static inline u16 mucse_desc_unused_rx(struct mucse_ring *ring) +{ + u16 ntc = ring->next_to_clean; + u16 ntu = ring->next_to_use; + + /* Keep M_RX_BUFFER_WRITE descriptors unused so the ring is not filled + * completely. Refill is attempted once at least this many descriptors + * are available. + */ + return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - M_RX_BUFFER_WRITE; +} + +static inline __le64 build_ctob(u32 vlan_cmd, u32 mac_ip_len, u32 size) +{ + return cpu_to_le64(((u64)vlan_cmd << 32) | ((u64)mac_ip_len << 16) | + ((u64)size)); +} + +static inline struct netdev_queue *txring_txq(const struct mucse_ring *ring) +{ + return netdev_get_tx_queue(ring->netdev, ring->queue_index); +} + +struct mucse_ring_container { + struct mucse_ring *ring; + u16 count; +}; + +struct mucse_q_vector { + struct mucse *mucse; + /* hardware interrupt vector number */ + int hw_vector; + struct mucse_ring_container rx, tx; + struct napi_struct napi; + struct timer_list rx_alloc_timer; + char name[IFNAMSIZ + 18]; + /* for dynamic allocation of rings associated with this q_vector */ + struct mucse_ring ring[] ____cacheline_internodealigned_in_smp; +}; + +#define MAX_Q_VECTORS 8 + +#define M_DEFAULT_TXD 512 +#define M_DEFAULT_RXD 512 +#define M_DEFAULT_TX_WORK 256 + +enum mucse_state_t { + __MUCSE_AXI_FAULT, }; struct mucse { struct net_device *netdev; struct pci_dev *pdev; struct mucse_hw hw; - struct mucse_stats stats; + struct mucse_ring *tx_ring[RNPGBE_MAX_QUEUES] + ____cacheline_aligned_in_smp; + struct mucse_ring *rx_ring[RNPGBE_MAX_QUEUES] + ____cacheline_aligned_in_smp; + struct mucse_queue_stats tx_stats[RNPGBE_MAX_QUEUES]; + struct mucse_queue_stats rx_stats[RNPGBE_MAX_QUEUES]; + struct mucse_q_vector *q_vector[MAX_Q_VECTORS]; + int tx_ring_item_count; + int tx_work_limit; + int num_tx_queues; + int num_q_vectors; + int rx_ring_item_count; + int num_rx_queues; + char mbx_name[32]; + bool mbx_irq_requested; + bool link_event_ready; + bool link_pending; + unsigned long state; + atomic_t mbx_irq_seq; + struct work_struct mbx_work; + struct delayed_work serv_task; + /* Protects event readiness, pending state, cached link state and + * driver-side link register updates. + */ + spinlock_t link_lock; }; int rnpgbe_get_permanent_mac(struct mucse_hw *hw, u8 *perm_addr); int rnpgbe_reset_hw(struct mucse_hw *hw); +int rnpgbe_setup_default_link(struct mucse_hw *hw); int rnpgbe_send_notify(struct mucse_hw *hw, bool enable, int mode); int rnpgbe_init_hw(struct mucse_hw *hw, int board_type); +int rnpgbe_set_rx_mode(struct net_device *netdev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc); +void rnpgbe_set_link(struct mucse_hw *hw, bool linkup); /* Device IDs */ #define PCI_VENDOR_ID_MUCSE 0x8848 @@ -68,4 +266,10 @@ int rnpgbe_init_hw(struct mucse_hw *hw, int board_type); #define mucse_hw_wr32(hw, reg, val) \ writel((val), (hw)->hw_addr + (reg)) +#define mucse_hw_rd32(hw, reg) \ + readl((hw)->hw_addr + (reg)) +#define mucse_ring_wr32(ring, reg, val) \ + writel((val), (ring)->ring_addr + (reg)) +#define mucse_ring_rd32(ring, reg) \ + readl((ring)->ring_addr + (reg)) #endif /* _RNPGBE_H */ diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_chip.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_chip.c index ebc7b3750157..dcc8233e024e 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_chip.c +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_chip.c @@ -4,12 +4,19 @@ #include #include #include +#include #include "rnpgbe.h" #include "rnpgbe_hw.h" #include "rnpgbe_mbx.h" #include "rnpgbe_mbx_fw.h" +#define RNPGBE_DEFAULT_ADV (RNPGBE_FW_ADV_10_FULL | \ + RNPGBE_FW_ADV_100_FULL | \ + RNPGBE_FW_ADV_1000_FULL | \ + RNPGBE_FW_ADV_10_HALF | \ + RNPGBE_FW_ADV_100_HALF) + /** * rnpgbe_get_permanent_mac - Get permanent mac * @hw: hw information structure @@ -53,6 +60,18 @@ int rnpgbe_reset_hw(struct mucse_hw *hw) return mucse_mbx_reset_hw(hw); } +/** + * rnpgbe_setup_default_link - Configure the default link settings + * @hw: hardware information structure + * + * Return: 0 on success, negative errno on failure + **/ +int rnpgbe_setup_default_link(struct mucse_hw *hw) +{ + return mucse_mbx_set_link(hw, RNPGBE_DEFAULT_ADV, true, 0, 0, + ETH_TP_MDI_AUTO); +} + /** * rnpgbe_send_notify - Echo fw status * @hw: hw information structure @@ -66,11 +85,17 @@ int rnpgbe_send_notify(struct mucse_hw *hw, int mode) { int err; - /* Keep switch struct to support more modes in the future */ + switch (mode) { case mucse_fw_powerup: err = mucse_mbx_powerup(hw, enable); break; + case mucse_fw_portup: + err = mucse_mbx_phyup(hw, enable); + break; + case mucse_fw_link_report_en: + err = mucse_mbx_link_report(hw, enable); + break; default: err = -EINVAL; } @@ -89,8 +114,12 @@ static void rnpgbe_init_n500(struct mucse_hw *hw) { struct mucse_mbx_info *mbx = &hw->mbx; + hw->ring_msix_base = hw->hw_addr + MUCSE_N500_RING_MSIX_BASE; + mbx->fwpf_ctrl_base = MUCSE_N500_FWPF_CTRL_BASE; mbx->fwpf_shm_base = MUCSE_N500_FWPF_SHM_BASE; + + hw->cycles_per_us = M_DEFAULT_N500_MHZ; } /** @@ -104,8 +133,12 @@ static void rnpgbe_init_n210(struct mucse_hw *hw) { struct mucse_mbx_info *mbx = &hw->mbx; + hw->ring_msix_base = hw->hw_addr + MUCSE_N210_RING_MSIX_BASE; + mbx->fwpf_ctrl_base = MUCSE_N210_FWPF_CTRL_BASE; mbx->fwpf_shm_base = MUCSE_N210_FWPF_SHM_BASE; + + hw->cycles_per_us = M_DEFAULT_N210_MHZ; } /** @@ -141,3 +174,135 @@ int rnpgbe_init_hw(struct mucse_hw *hw, int board_type) return 0; } + +static void rnpgbe_set_rar(struct mucse_hw *hw, u32 index, const u8 *addr) +{ + u32 old_low, old_high; + u32 rar_low, rar_high; + + /* The RAR stores the Ethernet address in reverse byte order. */ + rar_low = (u32)addr[5] | ((u32)addr[4] << 8) | + ((u32)addr[3] << 16) | ((u32)addr[2] << 24); + rar_high = (u32)addr[1] | ((u32)addr[0] << 8) | + RNPGBE_RX_RAR_VALID; + old_low = mucse_hw_rd32(hw, RNPGBE_RX_RAR_LOW(index)); + old_high = mucse_hw_rd32(hw, RNPGBE_RX_RAR_HIGH(index)); + if (old_low == rar_low && old_high == rar_high) + return; + + mucse_hw_wr32(hw, RNPGBE_RX_RAR_HIGH(index), + rar_high & ~RNPGBE_RX_RAR_VALID); + mucse_hw_wr32(hw, RNPGBE_RX_RAR_LOW(index), rar_low); + mucse_hw_wr32(hw, RNPGBE_RX_RAR_HIGH(index), rar_high); +} + +static void rnpgbe_clear_rar(struct mucse_hw *hw, u32 index) +{ + u32 rar_high; + + rar_high = mucse_hw_rd32(hw, RNPGBE_RX_RAR_HIGH(index)); + if (!(rar_high & RNPGBE_RX_RAR_VALID)) + return; + + mucse_hw_wr32(hw, RNPGBE_RX_RAR_HIGH(index), 0); + mucse_hw_wr32(hw, RNPGBE_RX_RAR_LOW(index), 0); +} + +int rnpgbe_set_rx_mode(struct net_device *netdev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) +{ + u32 mcast_hash[RNPGBE_RX_MCAST_HASH_ENTRIES] = {}; + u32 mcast_ctrl = RNPGBE_RX_UCAST_TABLE_EN; + u32 filter_ctrl = RNPGBE_RX_FILTER_BCAST; + struct netdev_hw_addr *ha; + struct mucse_hw *hw; + struct mucse *mucse; + int rar = 1; + u16 hash; + int i; + + mucse = netdev_priv(netdev); + hw = &mucse->hw; + + /* RAR 0 always holds the interface's primary unicast address. */ + rnpgbe_set_rar(hw, 0, netdev->dev_addr); + + netdev_hw_addr_list_for_each(ha, uc) { + if (rar == RNPGBE_RX_RAR_ENTRIES) { + filter_ctrl |= RNPGBE_RX_FILTER_UCAST_ALL; + break; + } + + rnpgbe_set_rar(hw, rar, ha->addr); + rar++; + } + + for (; rar < RNPGBE_RX_RAR_ENTRIES; rar++) + rnpgbe_clear_rar(hw, rar); + + if (netdev->flags & IFF_PROMISC) { + filter_ctrl |= RNPGBE_RX_FILTER_UCAST_ALL | + RNPGBE_RX_FILTER_MCAST_ALL; + } else if (netdev->flags & IFF_ALLMULTI) { + filter_ctrl |= RNPGBE_RX_FILTER_MCAST_ALL; + } else { + /* MTA type 0 uses ((addr[4] << 8) | addr[5]) & 0xfff. */ + netdev_hw_addr_list_for_each(ha, mc) { + hash = ((ha->addr[4] & 0xf) << 8) | ha->addr[5]; + mcast_hash[hash >> 5] |= BIT(hash & 0x1f); + } + + if (!netdev_hw_addr_list_empty(mc)) + mcast_ctrl |= RNPGBE_RX_MCAST_HASH_EN; + } + + /* Widen multicast filtering before invalidating hash entries. */ + if (filter_ctrl & RNPGBE_RX_FILTER_MCAST_ALL) + mucse_hw_wr32(hw, RNPGBE_RX_FILTER_CTRL, filter_ctrl); + + for (i = 0; i < RNPGBE_RX_MCAST_HASH_ENTRIES; i++) + mucse_hw_wr32(hw, RNPGBE_RX_MCAST_HASH(i), mcast_hash[i]); + + mucse_hw_wr32(hw, RNPGBE_RX_MCAST_CTRL, mcast_ctrl); + if (!(filter_ctrl & RNPGBE_RX_FILTER_MCAST_ALL)) + mucse_hw_wr32(hw, RNPGBE_RX_FILTER_CTRL, filter_ctrl); + + return 0; +} + +/** + * rnpgbe_set_link - Set the hardware link state + * @hw: hw information structure + * @linkup: link on or not + * + * Apply the firmware-reported link state to the GMAC. This helper is only + * called while processing a firmware link event. The caller must hold the + * containing struct mucse's link_lock. + * Firmware updates the speed and duplex fields before posting a link event + * and waits for the PF acknowledgment, so event handling completes this + * read-modify-write before firmware accesses GMAC_CONTROL again. + * + **/ +void rnpgbe_set_link(struct mucse_hw *hw, bool linkup) +{ + struct mucse *mucse = container_of(hw, struct mucse, hw); + u32 value; + + lockdep_assert_held(&mucse->link_lock); + value = mucse_hw_rd32(hw, GMAC_CONTROL); + + /* ndo_set_rx_mode_async() programs the chip-level filter. Keep the GMAC + * in receive-all mode so it does not discard frames accepted by that + * filter. + */ + if (linkup) { + mucse_hw_wr32(hw, GMAC_FRAME_FILTER, GMAC_RX_ALL); + value |= GMAC_CONTROL_RE; + mucse_hw_wr32(hw, GMAC_CONTROL, value); + } else { + value &= ~GMAC_CONTROL_RE; + mucse_hw_wr32(hw, GMAC_CONTROL, value); + mucse_hw_wr32(hw, GMAC_FRAME_FILTER, 0); + } +} diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_hw.h b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_hw.h index e77e6bc3d3e3..674d4d23c4b0 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_hw.h +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_hw.h @@ -4,14 +4,59 @@ #ifndef _RNPGBE_HW_H #define _RNPGBE_HW_H +#include + #define MUCSE_N500_FWPF_CTRL_BASE 0x28b00 #define MUCSE_N500_FWPF_SHM_BASE 0x2d000 +#define MUCSE_N500_RING_MSIX_BASE 0x28700 +#define M_DEFAULT_N500_MHZ 125 #define MUCSE_GBE_PFFW_MBX_CTRL_OFFSET 0x5500 #define MUCSE_GBE_FWPF_MBX_MASK_OFFSET 0x5700 #define MUCSE_N210_FWPF_CTRL_BASE 0x29400 #define MUCSE_N210_FWPF_SHM_BASE 0x2d900 +#define MUCSE_N210_RING_MSIX_BASE 0x29000 +#define M_DEFAULT_N210_MHZ 62 +#define RNPGBE_DMA_STATUS 0x0008 +#define TX_AXI_RW_EN 0xc +/* DMA_STATUS_REG[23:20]: tx_wr, tx_rd, rx_wr, rx_rd done status. */ +#define RNPGBE_DMA_TX_STATUS GENMASK_U32(23, 22) +#define RNPGBE_DMA_RX_STATUS GENMASK_U32(21, 20) +#define RX_AXI_RW_EN 0x03 +/* RNPGBE_LINK_ST is a driver-owned link-state snapshot consumed by firmware. + * Set the driver-state marker; all other driver status fields start clear. + */ +#define M_DEFAULT_ST 0xa0000000 +/* Firmware never reports 0xf in the speed field. */ +#define M_INVALID_ST (M_DEFAULT_ST | GENMASK_U32(11, 8)) +/* Driver-reported fields: 25:24 pause, 11:8 speed, 6 LLDP, 4 duplex, + * and 0 link up/down. M_DEFAULT_ST resets these fields so firmware reports + * the current hardware state again. + */ +#define RNPGBE_LINK_ST 0x000c #define RNPGBE_DMA_AXI_EN 0x0010 +#define RNPGBE_TX_MIN_PKT_LEN 33 +#define MUCSE_ETH_OFF(_n) (0x10000 + (_n)) +#define RNPGBE_RX_RAR_LOW(_n) MUCSE_ETH_OFF(0xa000 + 4 * (_n)) +#define RNPGBE_RX_RAR_HIGH(_n) MUCSE_ETH_OFF(0xa400 + 4 * (_n)) +#define RNPGBE_RX_MCAST_HASH(_n) MUCSE_ETH_OFF(0xac00 + 4 * (_n)) +#define RNPGBE_RX_FILTER_CTRL MUCSE_ETH_OFF(0x9110) +#define RNPGBE_RX_MCAST_CTRL MUCSE_ETH_OFF(0x9114) +#define RNPGBE_RX_RAR_VALID BIT(31) +#define RNPGBE_RX_FILTER_BCAST BIT(10) +#define RNPGBE_RX_FILTER_UCAST_ALL BIT(9) +#define RNPGBE_RX_FILTER_MCAST_ALL BIT(8) +#define RNPGBE_RX_MCAST_HASH_EN BIT(4) +#define RNPGBE_RX_UCAST_TABLE_EN BIT(3) +/* The final two of the 32 hardware RAR entries are reserved for NCSI. */ +#define RNPGBE_RX_RAR_ENTRIES 30 +#define RNPGBE_RX_MCAST_HASH_ENTRIES 128 + +#define MUCSE_GMAC_OFF(_n) (0x20000 + (_n)) +#define GMAC_CONTROL_RE 0x00000004 +#define GMAC_CONTROL MUCSE_GMAC_OFF(0) +#define GMAC_RX_ALL (BIT(31) | BIT(0)) +#define GMAC_FRAME_FILTER MUCSE_GMAC_OFF(0x4) #define RNPGBE_MAX_QUEUES 8 #endif /* _RNPGBE_HW_H */ diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.c new file mode 100644 index 000000000000..b7b0c45a0f66 --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.c @@ -0,0 +1,2262 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2020 - 2025 Mucse Corporation. */ + +#include +#include +#include +#include +#include +#include +#include + +#include "rnpgbe_lib.h" +#include "rnpgbe.h" +#include "rnpgbe_mbx_fw.h" + +static void rnpgbe_mbx_work(struct work_struct *work) +{ + struct mucse *mucse = container_of(work, struct mucse, mbx_work); + int irq_seq; + + irq_seq = atomic_read(&mucse->mbx_irq_seq); + mucse_fw_irq_handler(&mucse->hw); + + /* A mailbox interrupt while this work runs needs another invocation. */ + if (irq_seq != atomic_read(&mucse->mbx_irq_seq)) + queue_work(system_percpu_wq, &mucse->mbx_work); +} + +/** + * rnpgbe_msix_other - Other irq handler + * @irq: interrupt number + * @data: private data + * + * Return: IRQ_HANDLED + **/ +static irqreturn_t rnpgbe_msix_other(int irq, void *data) +{ + struct mucse *mucse = (struct mucse *)data; + + atomic_inc(&mucse->mbx_irq_seq); + queue_work(system_percpu_wq, &mucse->mbx_work); + + return IRQ_HANDLED; +} + +static void rnpgbe_irq_disable_queues(struct mucse_q_vector *q_vector) +{ + struct mucse_ring *ring; + + /* + * TX/RX pairs share interrupt control registers; update them through + * the TX ring list. + */ + mucse_for_each_ring(ring, q_vector->tx) { + writel(INT_VALID, ring->trig); + writel((RX_INT_MASK | TX_INT_MASK), ring->irq_mask); + } + /* flush posted writes to ensure hardware sees the mask */ + if (q_vector->tx.ring) + readl(q_vector->tx.ring->irq_mask); +} + +static void rnpgbe_irq_enable_queues(struct mucse_q_vector *q_vector) +{ + struct mucse_ring *ring; + + /* + * TX/RX pairs share interrupt control registers; update them through + * the TX ring list. + */ + mucse_for_each_ring(ring, q_vector->tx) { + writel(0, ring->irq_mask); + + /* Re-trigger hw to re-check events lost while masked. */ + writel(INT_VALID | TX_INT_MASK | RX_INT_MASK, ring->trig); + } +} + +/** + * rnpgbe_clean_tx_irq - Reclaim resources after transmit completes + * @q_vector: structure containing interrupt and ring information + * @tx_ring: tx ring to clean + * @napi_budget: Used to determine if we are in netpoll + * + * Return: true if cleaning completed within the budget, otherwise false + **/ +static bool rnpgbe_clean_tx_irq(struct mucse_q_vector *q_vector, + struct mucse_ring *tx_ring, + int napi_budget) +{ + int budget = q_vector->mucse->tx_work_limit; + u64 total_bytes = 0, total_packets = 0; + struct mucse *mucse = q_vector->mucse; + struct mucse_tx_buffer *tx_buffer; + struct rnpgbe_tx_desc *tx_desc; + int i = tx_ring->next_to_clean; + + tx_buffer = &tx_ring->tx_buffer_info[i]; + tx_desc = M_TX_DESC(tx_ring, i); + i -= tx_ring->count; + + do { + struct rnpgbe_tx_desc *eop_desc = tx_buffer->next_to_watch; + + /* if next_to_watch is not set then there is no work pending */ + if (!eop_desc) + break; + + /* prevent any other reads prior to eop_desc */ + dma_rmb(); + + /* if eop DD is not set pending work has not been completed */ + if (!(eop_desc->vlan_cmd & cpu_to_le32(M_TXD_STAT_DD))) + break; + /* clear next_to_watch to prevent false hangs */ + tx_buffer->next_to_watch = NULL; + total_bytes += tx_buffer->bytecount; + total_packets += tx_buffer->gso_segs; + napi_consume_skb(tx_buffer->skb, napi_budget); + if (tx_buffer->mapped_as_page) { + dma_unmap_page(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } else { + dma_unmap_single(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } + tx_buffer->skb = NULL; + dma_unmap_len_set(tx_buffer, len, 0); + + /* unmap remaining buffers */ + while (tx_desc != eop_desc) { + tx_buffer++; + tx_desc++; + i++; + if (unlikely(!i)) { + i -= tx_ring->count; + tx_buffer = tx_ring->tx_buffer_info; + tx_desc = M_TX_DESC(tx_ring, 0); + } + + /* unmap any remaining paged data */ + if (dma_unmap_len(tx_buffer, len)) { + dma_unmap_page(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + dma_unmap_len_set(tx_buffer, len, 0); + } + } + + /* move us one more past the eop_desc for start of next pkt */ + tx_buffer++; + tx_desc++; + i++; + if (unlikely(!i)) { + i -= tx_ring->count; + tx_buffer = tx_ring->tx_buffer_info; + tx_desc = M_TX_DESC(tx_ring, 0); + } + + prefetch(tx_desc); + budget--; + } while (likely(budget > 0)); + + i += tx_ring->count; + tx_ring->next_to_clean = i; + u64_stats_update_begin(&tx_ring->stats->syncp); + tx_ring->stats->bytes += total_bytes; + tx_ring->stats->packets += total_packets; + u64_stats_update_end(&tx_ring->stats->syncp); + +#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) + __netif_txq_completed_wake(txring_txq(tx_ring), + total_packets, total_bytes, + mucse_desc_unused(tx_ring), + TX_WAKE_THRESHOLD, + !netif_running(mucse->netdev)); + + return !!budget; +} + +static bool mucse_alloc_mapped_page(struct mucse_ring *rx_ring, + struct mucse_rx_buffer *bi) +{ + struct page *page = bi->page; + dma_addr_t dma; + + if (page) { + /* Buffer is being reused without going back through the + * page_pool. Do dma_sync for hw use. + */ + dma_sync_single_range_for_device(rx_ring->dev, bi->dma, + bi->page_offset, + RNPGBE_RX_DESC_DATA_LEN, + DMA_FROM_DEVICE); + return true; + } + + page = page_pool_dev_alloc_pages(rx_ring->page_pool); + if (unlikely(!page)) + return false; + dma = page_pool_get_dma_addr(page); + + bi->dma = dma; + bi->page = page; + bi->page_offset = RNPGBE_SKB_PAD; + + return true; +} + +static void mucse_update_rx_tail(struct mucse_ring *rx_ring, + u32 val) +{ + rx_ring->next_to_use = val; + writel(val, rx_ring->tail); +} + +/** + * rnpgbe_alloc_rx_buffers - Replace used receive buffers + * @rx_ring: ring to place buffers on + * @cleaned_count: number of buffers to replace + **/ +static void rnpgbe_alloc_rx_buffers(struct mucse_ring *rx_ring, + u16 cleaned_count) +{ + u64 fun_id = ((u64)(rx_ring->pfvfnum) << 56); + union rnpgbe_rx_desc *rx_desc; + u16 i = rx_ring->next_to_use; + struct mucse_rx_buffer *bi; + u64 addr; + /* nothing to do */ + if (!cleaned_count) + return; + + rx_desc = M_RX_DESC(rx_ring, i); + bi = &rx_ring->rx_buffer_info[i]; + i -= rx_ring->count; + + do { + if (!mucse_alloc_mapped_page(rx_ring, bi)) + break; + + addr = (u64)(bi->dma + bi->page_offset); + rx_desc->pkt_addr = cpu_to_le64(addr | fun_id); + /* clean dd */ + rx_desc->resv_cmd = 0; + rx_desc++; + bi++; + i++; + if (unlikely(!i)) { + rx_desc = M_RX_DESC(rx_ring, 0); + bi = rx_ring->rx_buffer_info; + i -= rx_ring->count; + } + cleaned_count--; + } while (cleaned_count); + + i += rx_ring->count; + + /* Publish the tail even when allocation stopped early, so a stale + * tail from a previous session cannot point hardware at a + * reallocated ring. + */ + dma_wmb(); + mucse_update_rx_tail(rx_ring, i); +} + +/** + * rnpgbe_get_buffer - Get the rx_buffer to be used + * @rx_ring: pointer to rx ring + * @skb: pointer skb for this packet + * @size: data size in this desc + * + * Return: rx_buffer. + **/ +static struct mucse_rx_buffer *rnpgbe_get_buffer(struct mucse_ring *rx_ring, + struct sk_buff **skb, + const unsigned int size) +{ + struct mucse_rx_buffer *rx_buffer; + + rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + *skb = rx_buffer->skb; + prefetchw(page_address(rx_buffer->page) + rx_buffer->page_offset); + /* we are reusing so sync this buffer for CPU use */ + dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, + rx_buffer->page_offset, size, + DMA_FROM_DEVICE); + + return rx_buffer; +} + +/** + * rnpgbe_add_rx_frag - Add non-linear data to the skb + * @rx_buffer: pointer to rx_buffer + * @skb: pointer skb for this packet + * @size: data size in this desc + **/ +static void rnpgbe_add_rx_frag(struct mucse_rx_buffer *rx_buffer, + struct sk_buff *skb, + unsigned int size) +{ + unsigned int truesize = PAGE_SIZE; + + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, + rx_buffer->page_offset, size, truesize); +} + +/** + * rnpgbe_build_skb - Try to build a skb based on rx_buffer + * @rx_buffer: pointer to rx_buffer + * @size: data size in this desc + * + * Return: skb for this rx_buffer + **/ +static struct sk_buff *rnpgbe_build_skb(struct mucse_rx_buffer *rx_buffer, + unsigned int size) +{ + void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; + unsigned int truesize = PAGE_SIZE; + struct sk_buff *skb; + + net_prefetch(va); + /* build an skb around the page buffer */ + skb = napi_build_skb(va - RNPGBE_SKB_PAD, truesize); + if (unlikely(!skb)) + return NULL; + /* update pointers within the skb to store the data */ + skb_reserve(skb, RNPGBE_SKB_PAD); + __skb_put(skb, size); + skb_mark_for_recycle(skb); + + return skb; +} + +/** + * rnpgbe_is_non_eop - Process handling of non-EOP buffers + * @rx_ring: rx ring being processed + * @rx_desc: rx descriptor for current buffer + * @skb: current socket buffer containing buffer in progress + * + * Advance next_to_clean. For a non-EOP descriptor, carry the skb into the + * next descriptor unless the fragment limit has been reached. In that case, + * free the skb and mark the remaining descriptors for drop. + * + * Return: true if the descriptor is not EOP. The skb may have been consumed + * when true is returned. + **/ +static bool rnpgbe_is_non_eop(struct mucse_ring *rx_ring, + union rnpgbe_rx_desc *rx_desc, + struct sk_buff *skb) +{ + u32 ntc = rx_ring->next_to_clean + 1; + + /* fetch, update, and store next to clean */ + ntc = (ntc < rx_ring->count) ? ntc : 0; + rx_ring->next_to_clean = ntc; + prefetch(M_RX_DESC(rx_ring, ntc)); + /* if we are the last buffer then there is nothing else to do */ + if (likely(rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP))) + return false; + if (skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS) { + /* place skb in next buffer to be received */ + rx_ring->rx_buffer_info[ntc].skb = skb; + } else { + rx_ring->drop_bytes = skb->len; + /* too much frags, force free */ + dev_kfree_skb_any(skb); + rx_ring->drop_reason = MUCSE_RX_DROP_RESOURCE; + } + /* we should clean it since we used all info in it */ + rx_desc->wb.cmd = 0; + + return true; +} + +/** + * rnpgbe_cleanup_headers - Correct corrupted or empty headers + * @skb: current socket buffer containing buffer in progress + * + * Return: true if an error was encountered and skb was freed. + **/ +static bool rnpgbe_cleanup_headers(struct sk_buff *skb) +{ + if (unlikely(!pskb_may_pull(skb, ETH_HLEN))) { + dev_kfree_skb_any(skb); + return true; + } + + /* if eth_skb_pad returns an error the skb was freed */ + if (eth_skb_pad(skb)) + return true; + + return false; +} + +/** + * rnpgbe_process_skb_fields - Set the RX queue and protocol fields + * @rx_ring: RX descriptor ring containing the queue information + * @skb: skb currently being received + * + * Records the RX queue that received the skb and sets its protocol from + * the Ethernet header. + **/ +static void rnpgbe_process_skb_fields(struct mucse_ring *rx_ring, + struct sk_buff *skb) +{ + struct net_device *dev = rx_ring->netdev; + + skb_record_rx_queue(skb, rx_ring->queue_index); + skb->protocol = eth_type_trans(skb, dev); +} + +/** + * rnpgbe_rx_alloc_retry - Retry RX buffer allocation + * @timer: RX allocation retry timer + * + * Schedules NAPI after RX buffer allocation fails. + **/ +static void rnpgbe_rx_alloc_retry(struct timer_list *timer) +{ + struct mucse_q_vector *q_vector = + timer_container_of(q_vector, timer, rx_alloc_timer); + + napi_schedule(&q_vector->napi); +} + +static bool rnpgbe_rx_refill_pending(struct mucse_q_vector *q_vector) +{ + struct mucse_ring *ring; + + mucse_for_each_ring(ring, q_vector->rx) { + if (mucse_desc_unused_rx(ring) >= M_RX_BUFFER_WRITE) + return true; + } + + return false; +} + +static void rnpgbe_schedule_rx_retry(struct mucse *mucse) +{ + struct mucse_q_vector *q_vector; + + for (int i = 0; i < mucse->num_q_vectors; i++) { + q_vector = mucse->q_vector[i]; + if (rnpgbe_rx_refill_pending(q_vector)) + mod_timer(&q_vector->rx_alloc_timer, + jiffies + msecs_to_jiffies(500)); + } +} + +/** + * rnpgbe_clean_rx_irq - Clean completed descriptors from Rx ring + * @q_vector: structure containing interrupt and ring information + * @rx_ring: rx descriptor ring to transact packets on + * @budget: total limit on number of packets to process + * + * rnpgbe_clean_rx_irq tries to check dd in desc, handle this desc + * if dd is set which means data is write-back by hw + * + * Return: amount of work completed. + **/ +static int rnpgbe_clean_rx_irq(struct mucse_q_vector *q_vector, + struct mucse_ring *rx_ring, + int budget) +{ + unsigned int total_rx_bytes = 0, total_rx_packets = 0; + u16 cleaned_count = mucse_desc_unused_rx(rx_ring); + unsigned int total_rx_length_errors = 0; + unsigned int total_rx_dropped = 0; + unsigned int work_done = 0; + + while (likely(work_done < budget)) { + struct mucse_rx_buffer *rx_buffer; + union rnpgbe_rx_desc *rx_desc; + struct sk_buff *skb; + unsigned int size; + + if (cleaned_count >= M_RX_BUFFER_WRITE) { + rnpgbe_alloc_rx_buffers(rx_ring, cleaned_count); + cleaned_count = 0; + } + rx_desc = M_RX_DESC(rx_ring, rx_ring->next_to_clean); + + if (!rnpgbe_test_staterr(rx_desc, M_RXD_STAT_DD)) + break; + + /* This memory barrier is needed to keep us from reading + * any other fields out of the rx_desc until we know the + * descriptor has been written back + */ + dma_rmb(); + /* Hardware guarantees the first descriptor of each packet is at + * least 33 bytes, including multi-descriptor packets. + * Multi-descriptor packets are only used for jumbo frames over + * 1536 bytes (M_DEFAULT_SG = 96). Each descriptor is at most + * 1536 bytes. Small packets use a single descriptor. + */ + size = le16_to_cpu(rx_desc->wb.len); + + if (unlikely(!size || size > RNPGBE_RX_DESC_DATA_LEN)) { + struct mucse_rx_buffer *err_rx_buffer; + u16 idx = rx_ring->next_to_clean; + + cleaned_count++; + /* Length errors override resource drops. */ + rx_ring->drop_reason = MUCSE_RX_DROP_LENGTH; + rx_ring->drop_bytes = 0; + + /* Free the partial skb from a previous non-EOP + * descriptor before advancing next_to_clean. + */ + err_rx_buffer = &rx_ring->rx_buffer_info[idx]; + if (unlikely(err_rx_buffer->skb)) { + dev_kfree_skb_any(err_rx_buffer->skb); + err_rx_buffer->skb = NULL; + } + + /* drop data until eop */ + if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) { + total_rx_length_errors++; + rx_ring->drop_reason = MUCSE_RX_DROP_NONE; + work_done++; + } + + rx_desc->wb.cmd = 0; + rx_ring->next_to_clean++; + if (rx_ring->next_to_clean >= rx_ring->count) + rx_ring->next_to_clean = 0; + continue; + } + + if (unlikely(rx_ring->drop_reason != MUCSE_RX_DROP_NONE)) { + cleaned_count++; + if (rx_ring->drop_reason == MUCSE_RX_DROP_RESOURCE) + rx_ring->drop_bytes += size; + + /* Count the packet once at EOP, using its final drop + * reason even across NAPI polls. + */ + if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) { + if (rx_ring->drop_reason == + MUCSE_RX_DROP_LENGTH) { + total_rx_length_errors++; + } else { + total_rx_bytes += rx_ring->drop_bytes; + total_rx_packets++; + total_rx_dropped++; + } + rx_ring->drop_reason = MUCSE_RX_DROP_NONE; + rx_ring->drop_bytes = 0; + work_done++; + } + + rx_desc->wb.cmd = 0; + rx_ring->next_to_clean++; + if (rx_ring->next_to_clean >= rx_ring->count) + rx_ring->next_to_clean = 0; + continue; + } + + rx_buffer = rnpgbe_get_buffer(rx_ring, &skb, size); + + if (skb) + rnpgbe_add_rx_frag(rx_buffer, skb, size); + else + skb = rnpgbe_build_skb(rx_buffer, size); + + if (!skb) { + cleaned_count++; + + /* drop until eop if multiple descriptors */ + if (rnpgbe_test_staterr(rx_desc, M_RXD_STAT_EOP)) { + total_rx_bytes += size; + total_rx_packets++; + total_rx_dropped++; + work_done++; + } else { + rx_ring->drop_bytes = size; + rx_ring->drop_reason = MUCSE_RX_DROP_RESOURCE; + } + + rx_desc->wb.cmd = 0; + rx_ring->next_to_clean++; + if (rx_ring->next_to_clean >= rx_ring->count) + rx_ring->next_to_clean = 0; + + continue; + } + + rx_buffer->page = NULL; + rx_buffer->skb = NULL; + cleaned_count++; + + if (rnpgbe_is_non_eop(rx_ring, rx_desc, skb)) + continue; + + /* verify the packet layout is correct */ + size = skb->len; + if (rnpgbe_cleanup_headers(skb)) { + if (size >= ETH_HLEN) { + total_rx_bytes += size; + total_rx_packets++; + } + /* we should clean it since we used all info in it */ + total_rx_dropped++; + rx_desc->wb.cmd = 0; + work_done++; + continue; + } + + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + rnpgbe_process_skb_fields(rx_ring, skb); + rx_desc->wb.cmd = 0; + napi_gro_receive(&q_vector->napi, skb); + /* Update packet statistics and NAPI work accounting. */ + total_rx_packets++; + work_done++; + } + + u64_stats_update_begin(&rx_ring->stats->syncp); + rx_ring->stats->packets += total_rx_packets; + rx_ring->stats->bytes += total_rx_bytes; + rx_ring->stats->length_errors += total_rx_length_errors; + atomic64_add(total_rx_dropped, &rx_ring->stats->dropped); + u64_stats_update_end(&rx_ring->stats->syncp); + + return work_done; +} + +/** + * rnpgbe_poll - NAPI polling callback + * @napi: structure for representing this polling device + * @budget: polling budget + * + * Clean completed TX and RX packets and complete NAPI polling when all + * queues are clean. + * + * Return: number of RX packets processed, or @budget if polling continues + **/ +static int rnpgbe_poll(struct napi_struct *napi, int budget) +{ + struct mucse_q_vector *q_vector = + container_of(napi, struct mucse_q_vector, napi); + bool clean_complete = true; + struct mucse_ring *ring; + bool refill_pending; + int per_ring_budget; + int work_done = 0; + + mucse_for_each_ring(ring, q_vector->tx) { + if (!rnpgbe_clean_tx_irq(q_vector, ring, budget)) + clean_complete = false; + } + + /* Exit if we are called by netpoll */ + if (unlikely(!budget)) + return 0; + + if (q_vector->rx.count > 1) + per_ring_budget = max(budget / q_vector->rx.count, 1); + else + per_ring_budget = budget; + + mucse_for_each_ring(ring, q_vector->rx) { + int cleaned = 0; + + cleaned = rnpgbe_clean_rx_irq(q_vector, ring, per_ring_budget); + work_done += cleaned; + if (cleaned >= per_ring_budget) + clean_complete = false; + } + + if (!clean_complete) + return budget; + + /* Read RX indices while this poll still owns the NAPI instance. */ + refill_pending = rnpgbe_rx_refill_pending(q_vector); + if (likely(napi_complete_done(napi, work_done))) { + if (refill_pending) + mod_timer(&q_vector->rx_alloc_timer, + jiffies + msecs_to_jiffies(500)); + rnpgbe_irq_enable_queues(q_vector); + } + + return work_done; +} + +/** + * rnpgbe_request_mbx_irq - Register mbx routine + * @mucse: pointer to private structure + * + * Register the dedicated mailbox handler on vector 0. + * + * Return: 0 on success, negative on failure + **/ +int rnpgbe_request_mbx_irq(struct mucse *mucse) +{ + struct pci_dev *pdev = mucse->pdev; + int err = 0; + + snprintf(mucse->mbx_name, sizeof(mucse->mbx_name), + "rnpgbe-mbx:%s", pci_name(pdev)); + INIT_WORK(&mucse->mbx_work, rnpgbe_mbx_work); + atomic_set(&mucse->mbx_irq_seq, 0); + + err = request_irq(pci_irq_vector(pdev, 0), rnpgbe_msix_other, 0, + mucse->mbx_name, mucse); + if (!err) + mucse->mbx_irq_requested = true; + + return err; +} + +/** + * rnpgbe_free_mbx_irq - Remove mbx routine + * @mucse: pointer to private structure + **/ +void rnpgbe_free_mbx_irq(struct mucse *mucse) +{ + struct pci_dev *pdev = mucse->pdev; + + if (mucse->mbx_irq_requested) { + mucse->mbx_irq_requested = false; + free_irq(pci_irq_vector(pdev, 0), mucse); + cancel_work_sync(&mucse->mbx_work); + } + cancel_delayed_work_sync(&mucse->serv_task); +} + +/** + * rnpgbe_set_num_queues - Set initial TX/RX queue counts + * @mucse: pointer to private structure + * + * Start with one TX queue and one RX queue. + **/ +static void rnpgbe_set_num_queues(struct mucse *mucse) +{ + mucse->num_tx_queues = 1; + mucse->num_rx_queues = 1; +} + +/** + * rnpgbe_set_interrupt_capability - Set MSI-X interrupt capability + * @mucse: pointer to private structure + * + * Allocate one MSI-X vector for the mailbox and one or more vectors for + * the data path. + * + * Return: 0 on success, negative on failure + **/ +static int rnpgbe_set_interrupt_capability(struct mucse *mucse) +{ + int v_budget; + + v_budget = min3(mucse->num_tx_queues, mucse->num_rx_queues, + MAX_Q_VECTORS); + v_budget = min_t(int, v_budget, num_online_cpus()); + /* add one vector for mbx */ + v_budget += 1; + + /* Vector 0 is reserved for mailbox events and must not share NAPI. */ + v_budget = pci_alloc_irq_vectors(mucse->pdev, 2, v_budget, + PCI_IRQ_MSIX); + if (v_budget < 0) + return v_budget; + + /* q_vectors do not include the mailbox vector. */ + mucse->num_q_vectors = v_budget - 1; + + return 0; +} + +/** + * mucse_add_ring - Add ring to ring container + * @ring: ring to be added + * @head: ring container + **/ +static void mucse_add_ring(struct mucse_ring *ring, + struct mucse_ring_container *head) +{ + ring->next = head->ring; + head->ring = ring; + head->count++; +} + +/** + * rnpgbe_alloc_q_vector - Allocate memory for a single interrupt vector + * @mucse: pointer to private structure + * @eth_queue_idx: queue_index idx for this q_vector + * @vector_idx: q_vector array index + * @r_idx: starting hardware ring index + * @r_count: number of TX/RX ring pairs + * @step: ring step + * + * Return: 0 on success. If allocation fails we return -ENOMEM. + **/ +static int rnpgbe_alloc_q_vector(struct mucse *mucse, + int eth_queue_idx, int vector_idx, int r_idx, + int r_count, int step) +{ + int rxr_idx = r_idx, txr_idx = r_idx; + struct mucse_hw *hw = &mucse->hw; + struct mucse_q_vector *q_vector; + int txr_count, rxr_count, idx; + struct mucse_ring *ring; + int ring_count; + + /* + * TX and RX rings are always allocated as pairs with the same hardware + * ring index. Interrupt control is shared by the pair, so the TX ring + * list is used to update the common registers. + */ + txr_count = r_count; + rxr_count = r_count; + ring_count = txr_count + rxr_count; + + q_vector = kzalloc_flex(*q_vector, ring, ring_count); + if (!q_vector) + return -ENOMEM; + + netif_napi_add(mucse->netdev, &q_vector->napi, rnpgbe_poll); + /* tie q_vector and mucse together */ + mucse->q_vector[vector_idx] = q_vector; + q_vector->mucse = mucse; + timer_setup(&q_vector->rx_alloc_timer, rnpgbe_rx_alloc_retry, 0); + q_vector->hw_vector = vector_idx; + /* Vector 0 is reserved for the mailbox. */ + q_vector->hw_vector++; + + ring = q_vector->ring; + + for (idx = 0; idx < txr_count; idx++) { + ring->dev = &mucse->pdev->dev; + mucse_add_ring(ring, &q_vector->tx); + ring->count = mucse->tx_ring_item_count; + ring->netdev = mucse->netdev; + ring->queue_index = eth_queue_idx + idx; + ring->rnpgbe_queue_idx = txr_idx; + ring->ring_addr = hw->hw_addr + RING_OFFSET(txr_idx); + ring->irq_mask = ring->ring_addr + RNPGBE_DMA_INT_MASK; + ring->trig = ring->ring_addr + RNPGBE_DMA_INT_TRIG; + ring->q_vector = q_vector; + ring->pfvfnum = hw->pfvfnum; + ring->stats = &mucse->tx_stats[ring->queue_index]; + mucse->tx_ring[ring->queue_index] = ring; + txr_idx += step; + ring++; + } + + for (idx = 0; idx < rxr_count; idx++) { + ring->dev = &mucse->pdev->dev; + mucse_add_ring(ring, &q_vector->rx); + ring->count = mucse->rx_ring_item_count; + ring->netdev = mucse->netdev; + ring->queue_index = eth_queue_idx + idx; + ring->rnpgbe_queue_idx = rxr_idx; + ring->ring_addr = hw->hw_addr + RING_OFFSET(rxr_idx); + ring->irq_mask = ring->ring_addr + RNPGBE_DMA_INT_MASK; + ring->trig = ring->ring_addr + RNPGBE_DMA_INT_TRIG; + ring->q_vector = q_vector; + ring->pfvfnum = hw->pfvfnum; + ring->stats = &mucse->rx_stats[ring->queue_index]; + mucse->rx_ring[ring->queue_index] = ring; + rxr_idx += step; + ring++; + } + + return 0; +} + +/** + * rnpgbe_free_q_vector - Free memory allocated for specific interrupt vector + * @mucse: pointer to private structure + * @vector_idx: q_vector array index + * + * This function frees the memory allocated to the q_vector. In addition if + * NAPI is enabled it will delete any references to the NAPI struct prior + * to freeing the q_vector. + **/ +static void rnpgbe_free_q_vector(struct mucse *mucse, int vector_idx) +{ + struct mucse_q_vector *q_vector = mucse->q_vector[vector_idx]; + struct mucse_ring *ring; + + timer_delete_sync(&q_vector->rx_alloc_timer); + + mucse_for_each_ring(ring, q_vector->tx) + mucse->tx_ring[ring->queue_index] = NULL; + mucse_for_each_ring(ring, q_vector->rx) + mucse->rx_ring[ring->queue_index] = NULL; + mucse->q_vector[vector_idx] = NULL; + netif_napi_del(&q_vector->napi); + kfree(q_vector); +} + +/** + * rnpgbe_alloc_q_vectors - Allocate memory for interrupt vectors + * @mucse: pointer to private structure + * + * Return: 0 on success, or -ENOMEM on allocation failure. + **/ +static int rnpgbe_alloc_q_vectors(struct mucse *mucse) +{ + int err, ring_cnt, v_remaining = mucse->num_q_vectors; + int r_remaining = min_t(int, mucse->num_tx_queues, + mucse->num_rx_queues); + int num_q_vectors = 0; + int eth_queue_idx = 0; + int vector_idx = 0; + int ring_step = 1; + int ring_idx = 0; + + for (; r_remaining > 0 && v_remaining > 0; v_remaining--) { + ring_cnt = DIV_ROUND_UP(r_remaining, v_remaining); + err = rnpgbe_alloc_q_vector(mucse, eth_queue_idx, + vector_idx, ring_idx, ring_cnt, + ring_step); + if (err) + goto err_free_q_vector; + ring_idx += ring_step * ring_cnt; + eth_queue_idx += ring_cnt; + r_remaining -= ring_cnt; + num_q_vectors++; + vector_idx++; + } + /* Record the number of queue vectors actually allocated */ + mucse->num_q_vectors = num_q_vectors; + mucse->num_tx_queues = eth_queue_idx; + mucse->num_rx_queues = eth_queue_idx; + + return 0; + +err_free_q_vector: + mucse->num_tx_queues = 0; + mucse->num_rx_queues = 0; + mucse->num_q_vectors = 0; + + while (vector_idx--) + rnpgbe_free_q_vector(mucse, vector_idx); + + return err; +} + +/** + * rnpgbe_reset_interrupt_capability - Reset irq capability setup + * @mucse: pointer to private structure + **/ +static void rnpgbe_reset_interrupt_capability(struct mucse *mucse) +{ + if (!mucse->pdev->msix_enabled) + return; + + pci_free_irq_vectors(mucse->pdev); +} + +/** + * rnpgbe_init_interrupt_scheme - Initialize the interrupt scheme + * @mucse: pointer to private structure + * + * Set up one TX/RX queue pair with a dedicated mailbox vector. + * + * Return: 0 on success, negative on failure + **/ +int rnpgbe_init_interrupt_scheme(struct mucse *mucse) +{ + int err; + + rnpgbe_set_num_queues(mucse); + + err = rnpgbe_set_interrupt_capability(mucse); + if (err) + return err; + + err = rnpgbe_alloc_q_vectors(mucse); + if (err) { + rnpgbe_reset_interrupt_capability(mucse); + return err; + } + + return 0; +} + +/** + * rnpgbe_free_q_vectors - Free memory allocated for interrupt vectors + * @mucse: pointer to private structure + * + * This function frees the memory allocated to the q_vectors. In addition if + * NAPI is enabled it will delete any references to the NAPI struct prior + * to freeing the q_vector. + **/ +static void rnpgbe_free_q_vectors(struct mucse *mucse) +{ + int vector_idx = mucse->num_q_vectors; + + mucse->num_q_vectors = 0; + + while (vector_idx--) + rnpgbe_free_q_vector(mucse, vector_idx); +} + +/** + * rnpgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings + * @mucse: pointer to private structure + * + * Clear interrupt specific resources and reset the structure + **/ +void rnpgbe_clear_interrupt_scheme(struct mucse *mucse) +{ + rnpgbe_free_q_vectors(mucse); + rnpgbe_reset_interrupt_capability(mucse); +} + +/** + * rnpgbe_msix_clean_rings - MSI-x interrupt handler for ring irq + * @irq: interrupt number + * @data: private data + * + * rnpgbe_msix_clean_rings handle irq from ring, start napi + * Return: IRQ_HANDLED + **/ +static irqreturn_t rnpgbe_msix_clean_rings(int irq, void *data) +{ + struct mucse_q_vector *q_vector = (struct mucse_q_vector *)data; + + rnpgbe_irq_disable_queues(q_vector); + napi_schedule_irqoff(&q_vector->napi); + + return IRQ_HANDLED; +} + +/** + * rnpgbe_request_irq - Register data-path MSI-X handlers + * @mucse: pointer to private structure + * + * Register interrupt handlers for the data-path MSI-X vectors. + * + * Return: 0 on success, negative value on failure + **/ +int rnpgbe_request_irq(struct mucse *mucse) +{ + struct net_device *netdev = mucse->netdev; + struct pci_dev *pdev = mucse->pdev; + struct mucse_q_vector *q_vector; + int err, i; + + for (i = 0; i < mucse->num_q_vectors; i++) { + q_vector = mucse->q_vector[i]; + + snprintf(q_vector->name, sizeof(q_vector->name), + "%s-%s-%d", netdev->name, "TxRx", i); + + err = request_irq(pci_irq_vector(pdev, i + 1), + rnpgbe_msix_clean_rings, 0, + q_vector->name, q_vector); + if (err) { + dev_err(&pdev->dev, "MSI-X req err %d: %d\n", + i + 1, err); + goto err_free_irqs; + } + } + + return 0; +err_free_irqs: + while (i--) { + q_vector = mucse->q_vector[i]; + synchronize_irq(pci_irq_vector(pdev, i + 1)); + free_irq(pci_irq_vector(pdev, i + 1), q_vector); + } + + return err; +} + +/** + * rnpgbe_free_irq - Free data-path MSI-X handlers + * @mucse: pointer to private structure + **/ +void rnpgbe_free_irq(struct mucse *mucse) +{ + struct pci_dev *pdev = mucse->pdev; + struct mucse_q_vector *q_vector; + + for (int i = 0; i < mucse->num_q_vectors; i++) { + q_vector = mucse->q_vector[i]; + if (!q_vector) + continue; + + free_irq(pci_irq_vector(pdev, i + 1), q_vector); + } +} + +/** + * rnpgbe_set_ring_vector - Set the ring_vector registers, + * mapping interrupt causes to vectors + * @mucse: pointer to private structure + * @queue: queue to map the corresponding interrupt to + * @vector: the vector num to map to the corresponding queue + * + */ +static void rnpgbe_set_ring_vector(struct mucse *mucse, + u8 queue, u8 vector) +{ + struct mucse_hw *hw = &mucse->hw; + u32 data; + + data = hw->pfvfnum << 24; + data |= (vector << 8); + data |= vector; + writel(data, hw->ring_msix_base + RING_VECTOR(queue)); +} + +/** + * rnpgbe_configure_msix - Configure MSI-X hardware + * @mucse: pointer to private structure + * + * rnpgbe_configure_msix sets up the hardware to properly generate MSI-X + * interrupts. + **/ +static void rnpgbe_configure_msix(struct mucse *mucse) +{ + struct mucse_q_vector *q_vector; + + for (int i = 0; i < mucse->num_q_vectors; i++) { + struct mucse_ring *ring; + + q_vector = mucse->q_vector[i]; + /* TX/RX pairs share vector registers. + * Update them through TX. + */ + mucse_for_each_ring(ring, q_vector->tx) { + rnpgbe_set_ring_vector(mucse, ring->rnpgbe_queue_idx, + q_vector->hw_vector); + } + } +} + +static void rnpgbe_irq_enable(struct mucse *mucse) +{ + for (int i = 0; i < mucse->num_q_vectors; i++) + rnpgbe_irq_enable_queues(mucse->q_vector[i]); +} + +/** + * rnpgbe_irq_disable - Mask queue interrupt generation on the NIC + * @mucse: board private structure + **/ +static void rnpgbe_irq_disable(struct mucse *mucse) +{ + struct pci_dev *pdev = mucse->pdev; + + for (int i = 0; i < mucse->num_q_vectors; i++) { + rnpgbe_irq_disable_queues(mucse->q_vector[i]); + synchronize_irq(pci_irq_vector(pdev, i + 1)); + } +} + +static void rnpgbe_napi_enable_all(struct mucse *mucse) +{ + for (int i = 0; i < mucse->num_q_vectors; i++) + napi_enable(&mucse->q_vector[i]->napi); +} + +static void rnpgbe_napi_disable_all(struct mucse *mucse) +{ + for (int i = 0; i < mucse->num_q_vectors; i++) + napi_disable(&mucse->q_vector[i]->napi); +} + +static void rnpgbe_stop_tx_ring(struct mucse_ring *tx_ring) +{ + if (!tx_ring->tx_buffer_info) + return; + + /* Stop hw. No new descriptors are fetched after TX_START=0. + * DMA for descriptors fetched before the stop may still be in flight. + */ + mucse_ring_wr32(tx_ring, RNPGBE_TX_START, 0); + /* Flush posted write to ensure hardware sees TX_START=0 */ + (void)mucse_ring_rd32(tx_ring, RNPGBE_TX_START); +} + +static int rnpgbe_wait_tx_dma_idle(struct mucse *mucse) +{ + struct mucse_hw *hw = &mucse->hw; + u32 dma_status; + int err; + + /* A timeout indicates a terminal AXI fault. Hardware stops all PCIe + * DMA requests in this state, including requests accepted before the + * fault. Teardown may therefore release mappings and descriptor memory. + * Recovery requires a chip-level reset. + */ + err = readl_poll_timeout(hw->hw_addr + RNPGBE_DMA_STATUS, + dma_status, + (dma_status & RNPGBE_DMA_TX_STATUS) == + RNPGBE_DMA_TX_STATUS, + 10, 100000); + if (err) { + set_bit(__MUCSE_AXI_FAULT, &mucse->state); + dev_err(&mucse->pdev->dev, + "TX DMA failed to quiesce, status %#x\n", dma_status); + } + + return err; +} + +static int rnpgbe_wait_rx_dma_idle(struct mucse *mucse) +{ + struct mucse_hw *hw = &mucse->hw; + u32 dma_status; + int err; + + /* A timeout indicates a terminal AXI fault. Hardware stops all PCIe + * DMA requests in this state, including requests accepted before the + * fault. Teardown may therefore release mappings and descriptor memory. + * Recovery requires a chip-level reset. + */ + err = readl_poll_timeout(hw->hw_addr + RNPGBE_DMA_STATUS, + dma_status, + (dma_status & RNPGBE_DMA_RX_STATUS) == + RNPGBE_DMA_RX_STATUS, + 10, 100000); + if (err) { + set_bit(__MUCSE_AXI_FAULT, &mucse->state); + dev_err(&mucse->pdev->dev, + "RX DMA failed to quiesce, status %#x\n", dma_status); + } + + return err; +} + +/** + * rnpgbe_stop_all_tx_rings - Stop TX DMA on all queues + * @mucse: board private structure + * + * Return: 0 when TX DMA was quiesced, negative errno otherwise + **/ +static int rnpgbe_stop_all_tx_rings(struct mucse *mucse) +{ + struct mucse_hw *hw = &mucse->hw; + u32 dma_axi_ctl; + int err = 0; + + for (int i = 0; i < mucse->num_tx_queues; i++) + rnpgbe_stop_tx_ring(mucse->tx_ring[i]); + + if (mucse->num_tx_queues && + !test_bit(__MUCSE_AXI_FAULT, &mucse->state)) + err = rnpgbe_wait_tx_dma_idle(mucse); + + dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN); + dma_axi_ctl &= ~TX_AXI_RW_EN; + mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl); + /* Flush the posted write before continuing. */ + (void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN); + + if (!test_bit(__MUCSE_AXI_FAULT, &mucse->state)) + return 0; + + return err ? err : -EIO; +} + +/** + * rnpgbe_clean_tx_ring - Free Tx Buffers + * @tx_ring: ring to be cleaned + **/ +static void rnpgbe_clean_tx_ring(struct mucse_ring *tx_ring) +{ + struct mucse_tx_buffer *tx_buffer; + u16 i = tx_ring->next_to_clean; + unsigned long size; + + /* ring already cleared, nothing to do */ + if (!tx_ring->tx_buffer_info) + return; + + tx_buffer = &tx_ring->tx_buffer_info[i]; + + while (i != tx_ring->next_to_use) { + struct rnpgbe_tx_desc *eop_desc, *tx_desc; + + dev_kfree_skb_any(tx_buffer->skb); + /* unmap skb header data */ + if (dma_unmap_len(tx_buffer, len)) { + if (tx_buffer->mapped_as_page) { + dma_unmap_page(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } else { + dma_unmap_single(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } + } + eop_desc = tx_buffer->next_to_watch; + tx_desc = M_TX_DESC(tx_ring, i); + /* unmap remaining buffers */ + while (tx_desc != eop_desc) { + tx_buffer++; + tx_desc++; + i++; + if (unlikely(i == tx_ring->count)) { + i = 0; + tx_buffer = tx_ring->tx_buffer_info; + tx_desc = M_TX_DESC(tx_ring, 0); + } + + /* unmap any remaining paged data */ + if (dma_unmap_len(tx_buffer, len)) + dma_unmap_page(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } + /* move us one more past the eop_desc for start of next pkt */ + tx_buffer++; + i++; + if (unlikely(i == tx_ring->count)) { + i = 0; + tx_buffer = tx_ring->tx_buffer_info; + } + } + + netdev_tx_reset_queue(txring_txq(tx_ring)); + size = sizeof(struct mucse_tx_buffer) * tx_ring->count; + memset(tx_ring->tx_buffer_info, 0, size); + /* Zero out the descriptor ring */ + memset(tx_ring->desc, 0, tx_ring->size); + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; +} + +/** + * rnpgbe_clean_all_tx_rings - Stop TX DMA and free Tx buffers for all queues + * @mucse: board private structure + **/ +void rnpgbe_clean_all_tx_rings(struct mucse *mucse) +{ + /* rnpgbe_stop_all_tx_rings() logs and latches any AXI fault. Hardware + * stops DMA requests in this state, so cleanup intentionally ignores + * an error and releases the mappings and descriptor memory. + */ + rnpgbe_stop_all_tx_rings(mucse); + + for (int i = 0; i < mucse->num_tx_queues; i++) + rnpgbe_clean_tx_ring(mucse->tx_ring[i]); +} + +static void rnpgbe_stop_rx_ring(struct mucse_ring *rx_ring) +{ + /* Stop hw. hardware design guarantees: + * - No new descriptors will be fetched after RX_START=0 + * - No DMA will be initiated for already-fetched descriptors + */ + mucse_ring_wr32(rx_ring, RNPGBE_RX_START, 0); + /* Flush posted write to ensure hardware sees RX_START=0 */ + (void)mucse_ring_rd32(rx_ring, RNPGBE_RX_START); +} + +/** + * rnpgbe_stop_all_rx_rings - Stop RX DMA for all queues + * @mucse: board private structure + * + * Return: 0 if RX DMA is quiesced and no AXI fault is latched, + * negative errno otherwise + **/ +static int rnpgbe_stop_all_rx_rings(struct mucse *mucse) +{ + struct mucse_hw *hw = &mucse->hw; + u32 dma_axi_ctl; + int err = 0; + + for (int i = 0; i < mucse->num_rx_queues; i++) + rnpgbe_stop_rx_ring(mucse->rx_ring[i]); + + if (mucse->num_rx_queues && + !test_bit(__MUCSE_AXI_FAULT, &mucse->state)) + err = rnpgbe_wait_rx_dma_idle(mucse); + + dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN); + dma_axi_ctl &= ~RX_AXI_RW_EN; + mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl); + /* Flush the posted write before continuing. */ + (void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN); + + if (!test_bit(__MUCSE_AXI_FAULT, &mucse->state)) + return 0; + + return err ? err : -EIO; +} + +/** + * rnpgbe_clean_rx_ring - Free Rx Buffers per Queue + * @rx_ring: ring to free buffers from + **/ +static void rnpgbe_clean_rx_ring(struct mucse_ring *rx_ring) +{ + struct mucse_rx_buffer *rx_buffer; + u16 i; + + /* ring already cleared, nothing to do */ + if (!rx_ring->rx_buffer_info) + return; + /* Free all the Rx ring sk_buffs */ + for (i = 0; i < rx_ring->count; i++) { + rx_buffer = &rx_ring->rx_buffer_info[i]; + + if (rx_buffer->skb) { + struct sk_buff *skb = rx_buffer->skb; + + dev_kfree_skb(skb); + rx_buffer->skb = NULL; + } + + if (rx_buffer->page) { + page_pool_put_full_page(rx_ring->page_pool, + rx_buffer->page, false); + rx_buffer->page = NULL; + } + } + + rx_ring->next_to_clean = 0; + rx_ring->next_to_use = 0; + rx_ring->drop_bytes = 0; + rx_ring->drop_reason = MUCSE_RX_DROP_NONE; +} + +/** + * rnpgbe_clean_all_rx_rings - Free Rx buffers for all queues + * @mucse: board private structure + **/ +static void rnpgbe_clean_all_rx_rings(struct mucse *mucse) +{ + for (int i = 0; i < mucse->num_rx_queues; i++) + rnpgbe_clean_rx_ring(mucse->rx_ring[i]); +} + +static void rnpgbe_cancel_rx_retry_timers(struct mucse *mucse) +{ + for (int i = 0; i < mucse->num_q_vectors; i++) + timer_delete_sync(&mucse->q_vector[i]->rx_alloc_timer); +} + +void rnpgbe_down(struct mucse *mucse) +{ + struct net_device *netdev = mucse->netdev; + struct mucse_hw *hw = &mucse->hw; + unsigned long flags; + int err; + + spin_lock_irqsave(&mucse->link_lock, flags); + mucse->link_event_ready = false; + mucse->link_pending = false; + hw->link = false; + hw->speed = 0; + hw->duplex = 0; + spin_unlock_irqrestore(&mucse->link_lock, flags); + cancel_delayed_work_sync(&mucse->serv_task); + + netif_carrier_off(netdev); + rnpgbe_irq_disable(mucse); + rnpgbe_napi_disable_all(mucse); + synchronize_net(); + rnpgbe_irq_disable(mucse); + netif_tx_disable(netdev); + rnpgbe_stop_all_rx_rings(mucse); + rnpgbe_cancel_rx_retry_timers(mucse); + rnpgbe_clean_all_tx_rings(mucse); + + err = rnpgbe_send_notify(hw, false, mucse_fw_link_report_en); + if (err) { + dev_warn(&hw->pdev->dev, "Send link report to hw failed %d\n", + err); + dev_warn(&hw->pdev->dev, "Fw will still report link event\n"); + } + + /* Firmware owns GMAC_CONTROL_RE and clears it while processing + * port-down. RX DMA is already quiesced, so no local GMAC update is + * needed before freeing the RX buffers. + */ + err = rnpgbe_send_notify(hw, false, mucse_fw_portup); + if (err) { + dev_warn(&hw->pdev->dev, "Send port down to hw failed %d\n", + err); + dev_warn(&hw->pdev->dev, "Port is not truly down\n"); + } + rnpgbe_clean_all_rx_rings(mucse); +} + +/** + * rnpgbe_up_complete - Final step for port up + * @mucse: pointer to private structure + * + * Return: 0 on success, negative errno if firmware setup fails + **/ +int rnpgbe_up_complete(struct mucse *mucse) +{ + struct net_device *netdev = mucse->netdev; + struct mucse_hw *hw = &mucse->hw; + unsigned long flags; + int err; + + rnpgbe_configure_msix(mucse); + rnpgbe_napi_enable_all(mucse); + rnpgbe_schedule_rx_retry(mucse); + rnpgbe_irq_enable(mucse); + netif_tx_start_all_queues(netdev); + err = rnpgbe_send_notify(hw, true, mucse_fw_portup); + if (err) { + dev_err(&hw->pdev->dev, + "Failed to notify firmware that port is up: %d\n", err); + return err; + } + /* Open link-event handling only after the data path is ready. Reset + * the snapshot at the same time so firmware reports the current state. + */ + spin_lock_irqsave(&mucse->link_lock, flags); + mucse->link_pending = false; + hw->link = false; + hw->speed = 0; + hw->duplex = 0; + mucse_hw_wr32(hw, RNPGBE_LINK_ST, M_DEFAULT_ST); + mucse->link_event_ready = true; + spin_unlock_irqrestore(&mucse->link_lock, flags); + /* Firmware checks RNPGBE_LINK_ST (driver's last-received link state) + * and only asserts LINK_CHANGE_EVT when it differs from the actual link + * state AND link_report_en is true. + */ + err = rnpgbe_send_notify(hw, true, mucse_fw_link_report_en); + if (err) { + dev_err(&hw->pdev->dev, + "Failed to enable firmware link reporting: %d\n", + err); + return err; + } + + return 0; +} + +/** + * rnpgbe_free_tx_resources - Free Tx Resources per Queue + * @tx_ring: tx descriptor ring for a specific queue + * + * Free all transmit software resources + **/ +static void rnpgbe_free_tx_resources(struct mucse_ring *tx_ring) +{ + vfree(tx_ring->tx_buffer_info); + tx_ring->tx_buffer_info = NULL; + /* if not set, then don't free */ + if (!tx_ring->desc) + return; + + dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, + tx_ring->dma); + tx_ring->desc = NULL; +} + +/** + * rnpgbe_setup_tx_resources - allocate Tx resources (Descriptors) + * @tx_ring: tx descriptor ring (for a specific queue) to setup + * + * Return: 0 on success, negative on failure + **/ +static int rnpgbe_setup_tx_resources(struct mucse_ring *tx_ring) +{ + struct device *dev = tx_ring->dev; + int size; + + size = sizeof(struct mucse_tx_buffer) * tx_ring->count; + + tx_ring->tx_buffer_info = vzalloc(size); + if (!tx_ring->tx_buffer_info) + goto err_return; + /* round up to nearest 4K */ + tx_ring->size = tx_ring->count * sizeof(struct rnpgbe_tx_desc); + tx_ring->size = ALIGN(tx_ring->size, 4096); + tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, &tx_ring->dma, + GFP_KERNEL); + if (!tx_ring->desc) + goto err_free_buffer; + + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; + + return 0; + +err_free_buffer: + vfree(tx_ring->tx_buffer_info); +err_return: + tx_ring->tx_buffer_info = NULL; + return -ENOMEM; +} + +/** + * rnpgbe_configure_tx_ring - Configure Tx ring after Reset + * @mucse: pointer to private structure + * @ring: structure containing ring specific data + * + * Configure the Tx descriptor ring after a reset. + **/ +static void rnpgbe_configure_tx_ring(struct mucse *mucse, + struct mucse_ring *ring) +{ + struct mucse_hw *hw = &mucse->hw; + + mucse_ring_wr32(ring, RNPGBE_TX_BASE_ADDR_LO, (u32)ring->dma); + mucse_ring_wr32(ring, RNPGBE_TX_BASE_ADDR_HI, + (u32)(((u64)ring->dma) >> 32) | + ((u32)hw->pfvfnum << 24)); + mucse_ring_wr32(ring, RNPGBE_TX_LEN, ring->count); + ring->next_to_clean = mucse_ring_rd32(ring, RNPGBE_TX_HEAD) % + ring->count; + ring->next_to_use = ring->next_to_clean; + ring->tail = ring->ring_addr + RNPGBE_TX_TAIL; + writel(ring->next_to_use, ring->tail); + mucse_ring_wr32(ring, RNPGBE_TX_FETCH_CTRL, M_DEFAULT_TX_FETCH); + mucse_ring_wr32(ring, RNPGBE_TX_INT_TIMER, + M_DEFAULT_INT_TIMER * hw->cycles_per_us); + mucse_ring_wr32(ring, RNPGBE_TX_INT_PKTCNT, M_DEFAULT_INT_PKTCNT); +} + +/** + * rnpgbe_configure_tx - Configure Transmit Unit after Reset + * @mucse: pointer to private structure + * + * Configure the Tx DMA after a reset. + * + * Return: 0 on success, negative errno if TX DMA did not quiesce + **/ +int rnpgbe_configure_tx(struct mucse *mucse) +{ + struct mucse_hw *hw = &mucse->hw; + u32 i, dma_axi_ctl; + int err; + + err = rnpgbe_stop_all_tx_rings(mucse); + if (err) + return err; + + for (i = 0; i < mucse->num_tx_queues; i++) + rnpgbe_configure_tx_ring(mucse, mucse->tx_ring[i]); + + /* Ensure all ring configuration is visible before enabling TX DMA. */ + wmb(); + dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN); + dma_axi_ctl |= TX_AXI_RW_EN; + mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl); + /* Ensure TX_AXI_RW_EN is visible before starting the rings. */ + (void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN); + + for (i = 0; i < mucse->num_tx_queues; i++) + mucse_ring_wr32(mucse->tx_ring[i], RNPGBE_TX_START, 1); + + return 0; +} + +/** + * rnpgbe_setup_all_tx_resources - allocate all queues Tx resources + * @mucse: pointer to private structure + * + * Allocate memory for tx_ring. + * + * Return: 0 on success, negative on failure + **/ +int rnpgbe_setup_all_tx_resources(struct mucse *mucse) +{ + int i, err = 0; + + for (i = 0; i < mucse->num_tx_queues; i++) { + err = rnpgbe_setup_tx_resources(mucse->tx_ring[i]); + if (!err) + continue; + + goto err_free_res; + } + + return 0; +err_free_res: + while (i--) + rnpgbe_free_tx_resources(mucse->tx_ring[i]); + return err; +} + +/** + * rnpgbe_free_all_tx_resources - Free Tx Resources for All Queues + * @mucse: pointer to private structure + * + * Free all transmit software resources + **/ +void rnpgbe_free_all_tx_resources(struct mucse *mucse) +{ + for (int i = 0; i < (mucse->num_tx_queues); i++) + rnpgbe_free_tx_resources(mucse->tx_ring[i]); +} + +static int rnpgbe_tx_map(struct mucse_ring *tx_ring, + struct mucse_tx_buffer *first, u32 mac_ip_len, + u32 tx_flags) +{ + struct mucse_tx_buffer *tx_buffer; + struct rnpgbe_tx_desc *tx_desc; + struct skb_shared_info *shinfo; + unsigned int data_len, size; + struct sk_buff *skb; + skb_frag_t *frag; + dma_addr_t dma; + int nr_frags; + int frag_idx; + /* Hardware requires this in the upper 8 bits of + * the descriptor address + */ + u64 fun_id; + u16 i; + + skb = first->skb; + shinfo = skb_shinfo(skb); + fun_id = (u64)tx_ring->pfvfnum << 56; + nr_frags = shinfo->nr_frags; + i = tx_ring->next_to_use; + frag_idx = 0; + tx_desc = M_TX_DESC(tx_ring, i); + size = skb_headlen(skb); + data_len = skb->data_len; + + if (size) { + dma = dma_map_single(tx_ring->dev, skb->data, size, + DMA_TO_DEVICE); + first->mapped_as_page = false; + } else if (data_len) { + while (frag_idx < nr_frags && + !skb_frag_size(&shinfo->frags[frag_idx])) + frag_idx++; + + if (frag_idx == nr_frags) + goto err_unmap; + + frag = &shinfo->frags[frag_idx]; + size = skb_frag_size(frag); + + dma = skb_frag_dma_map(tx_ring->dev, frag, 0, + size, DMA_TO_DEVICE); + first->mapped_as_page = true; + data_len -= size; + frag_idx++; + } else { + goto err_unmap; + } + + tx_buffer = first; + + dma_unmap_len_set(tx_buffer, len, 0); + dma_unmap_addr_set(tx_buffer, dma, 0); + + for (;; frag_idx++) { + if (dma_mapping_error(tx_ring->dev, dma)) + goto err_unmap; + + /* record length, and DMA address */ + dma_unmap_len_set(tx_buffer, len, size); + dma_unmap_addr_set(tx_buffer, dma, dma); + + tx_desc->pkt_addr = cpu_to_le64(dma | fun_id); + + while (unlikely(size > M_MAX_DATA_PER_TXD)) { + tx_desc->vlan_cmd_bsz = build_ctob(tx_flags, + mac_ip_len, + M_MAX_DATA_PER_TXD); + i++; + tx_desc++; + if (i == tx_ring->count) { + tx_desc = M_TX_DESC(tx_ring, 0); + i = 0; + } + + tx_buffer = &tx_ring->tx_buffer_info[i]; + dma_unmap_len_set(tx_buffer, len, 0); + dma += M_MAX_DATA_PER_TXD; + size -= M_MAX_DATA_PER_TXD; + tx_desc->pkt_addr = cpu_to_le64(dma | fun_id); + } + + if (likely(!data_len)) + break; + tx_desc->vlan_cmd_bsz = build_ctob(tx_flags, mac_ip_len, size); + i++; + tx_desc++; + if (i == tx_ring->count) { + tx_desc = M_TX_DESC(tx_ring, 0); + i = 0; + } + + while (frag_idx < nr_frags && + !skb_frag_size(&shinfo->frags[frag_idx])) + frag_idx++; + + if (frag_idx == nr_frags) + goto err_unmap; + + frag = &shinfo->frags[frag_idx]; + size = skb_frag_size(frag); + data_len -= size; + dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, + DMA_TO_DEVICE); + tx_buffer = &tx_ring->tx_buffer_info[i]; + tx_buffer->mapped_as_page = true; + } + + /* write last descriptor with RS and EOP bits */ + tx_desc->vlan_cmd_bsz = build_ctob(tx_flags | M_TXD_CMD_EOP | + M_TXD_CMD_RS, + mac_ip_len, size); + + /* Force memory writes to complete before letting h/w know there + * are new descriptors to fetch. (Only applicable for weak-ordered + * memory model archs, such as IA-64). + * + * We also need this memory barrier to make certain all of the + * status bits have been updated before next_to_watch is written. + */ + dma_wmb(); + /* set next_to_watch value indicating a packet is present */ + first->next_to_watch = tx_desc; + i++; + if (i == tx_ring->count) + i = 0; + tx_ring->next_to_use = i; + skb_tx_timestamp(skb); + netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); + /* notify HW of packet */ + writel(i, tx_ring->tail); + + return 0; +err_unmap: + for (;;) { + tx_buffer = &tx_ring->tx_buffer_info[i]; + if (dma_unmap_len(tx_buffer, len)) { + if (tx_buffer->mapped_as_page) { + dma_unmap_page(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } else { + dma_unmap_single(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + } + } + dma_unmap_len_set(tx_buffer, len, 0); + dma_unmap_addr_set(tx_buffer, dma, 0); + if (tx_buffer == first) + break; + if (i == 0) + i += tx_ring->count; + i--; + } + dev_kfree_skb_any(first->skb); + first->skb = NULL; + tx_ring->next_to_use = i; + + return -ENOMEM; +} + +netdev_tx_t rnpgbe_xmit_frame_ring(struct sk_buff *skb, + struct mucse_ring *tx_ring) +{ + /* hw requires non-zero mac_ip_len even without offload; + * Will be updated per-packet when offload is added + */ + u32 mac_ip_len = M_DEFAULT_MAC_IP_LEN; + struct mucse_tx_buffer *first; + u32 tx_flags = 0; + unsigned short f; + u16 count; + + count = TXD_USE_COUNT(skb_headlen(skb)); + for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) { + skb_frag_t *frag_temp = &skb_shinfo(skb)->frags[f]; + + count += TXD_USE_COUNT(skb_frag_size(frag_temp)); + } + + if (!netif_txq_maybe_stop(txring_txq(tx_ring), + mucse_desc_unused(tx_ring), + count + RESV_DESC_NEEDED, + count + RESV_DESC_NEEDED)) + return NETDEV_TX_BUSY; + + /* record the location of the first descriptor for this packet */ + first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; + first->skb = skb; + first->bytecount = skb->len; + first->gso_segs = 1; + + if (rnpgbe_tx_map(tx_ring, first, mac_ip_len, tx_flags)) { + atomic64_inc(&tx_ring->stats->dropped); + + goto out; + } + + /* NETDEV_TX_BUSY is expensive. So stop advancing the TX queue. */ + netif_txq_maybe_stop(txring_txq(tx_ring), + mucse_desc_unused(tx_ring), + DESC_NEEDED, DESC_NEEDED); +out: + return NETDEV_TX_OK; +} + +/** + * rnpgbe_get_stats64 - Get stats for this netdev + * @netdev: network interface device structure + * @stats: stats data + **/ +void rnpgbe_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats) +{ + struct mucse *mucse = netdev_priv(netdev); + int i; + + for (i = 0; i < RNPGBE_MAX_QUEUES; i++) { + unsigned int start; + u64 bytes, packets, dropped; + struct mucse_queue_stats *queue_stats = &mucse->tx_stats[i]; + + do { + start = u64_stats_fetch_begin(&queue_stats->syncp); + packets = queue_stats->packets; + bytes = queue_stats->bytes; + } while (u64_stats_fetch_retry(&queue_stats->syncp, start)); + dropped = atomic64_read(&queue_stats->dropped); + + stats->tx_packets += packets; + stats->tx_dropped += dropped; + stats->tx_bytes += bytes; + } + + for (i = 0; i < RNPGBE_MAX_QUEUES; i++) { + unsigned int start; + u64 bytes, packets, dropped, length_errors; + struct mucse_queue_stats *queue_stats = &mucse->rx_stats[i]; + + do { + start = u64_stats_fetch_begin(&queue_stats->syncp); + packets = queue_stats->packets; + bytes = queue_stats->bytes; + length_errors = queue_stats->length_errors; + } while (u64_stats_fetch_retry(&queue_stats->syncp, start)); + dropped = atomic64_read(&queue_stats->dropped); + + stats->rx_packets += packets; + stats->rx_dropped += dropped; + stats->rx_bytes += bytes; + stats->rx_errors += length_errors; + stats->rx_length_errors += length_errors; + } +} + +static int mucse_alloc_page_pool(struct mucse_ring *rx_ring) +{ + struct page_pool_params pp_params = { + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .order = 0, + .pool_size = rx_ring->count, + .nid = dev_to_node(rx_ring->dev), + .dev = rx_ring->dev, + .napi = &rx_ring->q_vector->napi, + .dma_dir = DMA_FROM_DEVICE, + .offset = RNPGBE_SKB_PAD, + .max_len = RNPGBE_RX_DESC_DATA_LEN, + .netdev = rx_ring->netdev, + .queue_idx = rx_ring->queue_index, + }; + int ret = 0; + + rx_ring->page_pool = page_pool_create(&pp_params); + if (IS_ERR(rx_ring->page_pool)) { + ret = PTR_ERR(rx_ring->page_pool); + rx_ring->page_pool = NULL; + } + + return ret; +} + +/** + * rnpgbe_setup_rx_resources - allocate Rx resources (Descriptors) + * @rx_ring: rx descriptor ring (for a specific queue) to setup + * + * Return: 0 on success, negative on failure + **/ +static int rnpgbe_setup_rx_resources(struct mucse_ring *rx_ring) +{ + struct device *dev = rx_ring->dev; + int size; + + size = sizeof(struct mucse_rx_buffer) * rx_ring->count; + + rx_ring->rx_buffer_info = vzalloc(size); + + if (!rx_ring->rx_buffer_info) + goto err_return; + /* Round up to nearest 4K */ + rx_ring->size = rx_ring->count * sizeof(union rnpgbe_rx_desc); + rx_ring->size = ALIGN(rx_ring->size, 4096); + rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, &rx_ring->dma, + GFP_KERNEL); + if (!rx_ring->desc) + goto err_free_buffer; + + rx_ring->next_to_clean = 0; + rx_ring->next_to_use = 0; + + if (mucse_alloc_page_pool(rx_ring)) + goto err_free_desc; + + return 0; +err_free_desc: + dma_free_coherent(dev, rx_ring->size, rx_ring->desc, + rx_ring->dma); + rx_ring->desc = NULL; +err_free_buffer: + vfree(rx_ring->rx_buffer_info); +err_return: + rx_ring->rx_buffer_info = NULL; + return -ENOMEM; +} + +/** + * rnpgbe_free_rx_resources - Free Rx Resources + * @rx_ring: ring to clean the resources from + * + * RX DMA must be quiesced and all ring-held buffers released before this + * function is called. + **/ +static void rnpgbe_free_rx_resources(struct mucse_ring *rx_ring) +{ + vfree(rx_ring->rx_buffer_info); + rx_ring->rx_buffer_info = NULL; + /* if not set, then don't free */ + if (!rx_ring->desc) + return; + + dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, + rx_ring->dma); + rx_ring->desc = NULL; + if (rx_ring->page_pool) { + page_pool_destroy(rx_ring->page_pool); + rx_ring->page_pool = NULL; + } +} + +/** + * rnpgbe_setup_all_rx_resources - allocate all queues Rx resources + * @mucse: pointer to private structure + * + * Return: 0 on success, negative on failure + **/ +int rnpgbe_setup_all_rx_resources(struct mucse *mucse) +{ + int i, err = 0; + + for (i = 0; i < mucse->num_rx_queues; i++) { + err = rnpgbe_setup_rx_resources(mucse->rx_ring[i]); + if (!err) + continue; + + goto err_setup_rx; + } + + return 0; +err_setup_rx: + while (i--) + rnpgbe_free_rx_resources(mucse->rx_ring[i]); + return err; +} + +/** + * rnpgbe_free_all_rx_resources - Free Rx Resources for All Queues + * @mucse: pointer to private structure + * + * Free all receive software resources + **/ +void rnpgbe_free_all_rx_resources(struct mucse *mucse) +{ + for (int i = 0; i < mucse->num_rx_queues; i++) + rnpgbe_free_rx_resources(mucse->rx_ring[i]); +} + +/** + * rnpgbe_configure_rx_ring - Configure Rx ring info to hw + * @mucse: pointer to private structure + * @ring: structure containing ring specific data + * + * Configure the Rx descriptor ring after a reset. + **/ +static void rnpgbe_configure_rx_ring(struct mucse *mucse, + struct mucse_ring *ring) +{ + struct mucse_hw *hw = &mucse->hw; + + /* Set the descriptor registers after RX DMA has quiesced. */ + mucse_ring_wr32(ring, RNPGBE_RX_BASE_ADDR_LO, (u32)ring->dma); + mucse_ring_wr32(ring, RNPGBE_RX_BASE_ADDR_HI, + (u32)((u64)ring->dma >> 32) | + ((u32)hw->pfvfnum << 24)); + mucse_ring_wr32(ring, RNPGBE_RX_LEN, ring->count); + ring->tail = ring->ring_addr + RNPGBE_RX_TAIL; + ring->next_to_clean = mucse_ring_rd32(ring, RNPGBE_RX_HEAD) % + ring->count; + ring->next_to_use = ring->next_to_clean; + ring->drop_bytes = 0; + ring->drop_reason = MUCSE_RX_DROP_NONE; + mucse_ring_wr32(ring, RNPGBE_RX_SG_LEN, M_DEFAULT_SG); + mucse_ring_wr32(ring, RNPGBE_RX_FETCH, M_DEFAULT_RX_FETCH); + mucse_ring_wr32(ring, RNPGBE_RX_TIMEOUT_TH, 0); + mucse_ring_wr32(ring, RNPGBE_RX_INT_TIMER, + M_DEFAULT_INT_TIMER_R * hw->cycles_per_us); + mucse_ring_wr32(ring, RNPGBE_RX_INT_PKTCNT, M_DEFAULT_RX_INT_PKTCNT); + /* Fill the ring before RX DMA is enabled. Batch-sized allocation + * shortfalls are retried after NAPI is enabled; smaller shortfalls are + * replenished once traffic frees a complete refill batch. + */ + rnpgbe_alloc_rx_buffers(ring, mucse_desc_unused_rx(ring)); +} + +/** + * rnpgbe_configure_rx - Configure Receive Unit after Reset + * @mucse: pointer to private structure + * + * Configure the Rx unit after a reset. + * + * Return: 0 on success, negative errno if RX DMA does not quiesce or + * an AXI fault is already latched + **/ +int rnpgbe_configure_rx(struct mucse *mucse) +{ + struct mucse_hw *hw = &mucse->hw; + u32 dma_axi_ctl; + int err; + + err = rnpgbe_stop_all_rx_rings(mucse); + if (err) + return err; + + for (int i = 0; i < mucse->num_rx_queues; i++) + rnpgbe_configure_rx_ring(mucse, mucse->rx_ring[i]); + + /* Ensure all ring configuration is visible before enabling RX DMA. */ + wmb(); + dma_axi_ctl = mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN); + dma_axi_ctl |= RX_AXI_RW_EN; + mucse_hw_wr32(hw, RNPGBE_DMA_AXI_EN, dma_axi_ctl); + /* Ensure RX_AXI_RW_EN is visible before starting the rings. */ + (void)mucse_hw_rd32(hw, RNPGBE_DMA_AXI_EN); + + for (int i = 0; i < mucse->num_rx_queues; i++) + mucse_ring_wr32(mucse->rx_ring[i], RNPGBE_RX_START, 1); + + return 0; +} + +/** + * rnpgbe_process_link_event - Consume a pending link event + * @mucse: pointer to the device private structure + * @link: link status snapshot + * @speed: link speed snapshot + * @duplex: link duplex snapshot + * + * Return: true if a link event was consumed, false otherwise + **/ +static bool rnpgbe_process_link_event(struct mucse *mucse, bool *link, + int *speed, u8 *duplex) +{ + struct mucse_hw *hw = &mucse->hw; + unsigned long flags; + + spin_lock_irqsave(&mucse->link_lock, flags); + if (!mucse->link_event_ready || !mucse->link_pending) { + spin_unlock_irqrestore(&mucse->link_lock, flags); + return false; + } + mucse->link_pending = false; + *link = hw->link; + *speed = hw->speed; + *duplex = hw->duplex; + spin_unlock_irqrestore(&mucse->link_lock, flags); + + return true; +} + +/** + * rnpgbe_link_is_up - Update netif_carrier status and + * print link up message + * @mucse: pointer to the device private structure + * @speed: link speed snapshot + * @duplex: link duplex snapshot + **/ +static void rnpgbe_link_is_up(struct mucse *mucse, int speed, u8 duplex) +{ + struct net_device *netdev = mucse->netdev; + + /* Only continue if link was previously down */ + if (netif_carrier_ok(netdev)) + return; + + netdev_info(netdev, "NIC Link is Up %d Mbps, %s Duplex\n", + speed, duplex ? "Full" : "Half"); + netif_carrier_on(netdev); +} + +/** + * rnpgbe_link_is_down - Update netif_carrier status and + * print link down message + * @mucse: pointer to the private structure + **/ +static void rnpgbe_link_is_down(struct mucse *mucse) +{ + struct net_device *netdev = mucse->netdev; + + /* Only continue if link was up previously */ + if (!netif_carrier_ok(netdev)) + return; + netdev_info(netdev, "NIC Link is Down\n"); + netif_carrier_off(netdev); +} + +/** + * rnpgbe_process_link_subtask - Process a link-state update + * @mucse: pointer to the device private structure + **/ +static void rnpgbe_process_link_subtask(struct mucse *mucse) +{ + bool link; + int speed; + u8 duplex; + + if (!rnpgbe_process_link_event(mucse, &link, &speed, &duplex)) + return; + + if (link) + rnpgbe_link_is_up(mucse, speed, duplex); + else + rnpgbe_link_is_down(mucse); +} + +/** + * rnpgbe_service_task - Manages and runs subtasks + * @work: pointer to work_struct containing our data + **/ +void rnpgbe_service_task(struct work_struct *work) +{ + struct mucse *mucse = container_of(work, struct mucse, serv_task.work); + + rnpgbe_process_link_subtask(mucse); +} diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.h b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.h new file mode 100644 index 000000000000..8ebaa9534dac --- /dev/null +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_lib.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2020 - 2025 Mucse Corporation. */ + +#ifndef _RNPGBE_LIB_H +#define _RNPGBE_LIB_H + +#include +#include +#include + +struct mucse; +struct mucse_hw; +struct mucse_ring; + +#define RING_OFFSET(n) (0x1000 + 0x100 * (n)) +#define RNPGBE_RX_START 0x10 +#define RNPGBE_TX_START 0x18 +#define RNPGBE_DMA_INT_MASK 0x24 +#define TX_INT_MASK BIT(1) +#define RX_INT_MASK BIT(0) +#define INT_VALID (BIT(16) | BIT(17)) +#define RNPGBE_DMA_INT_TRIG 0x2c /* lost-interrupt recovery trigger */ +#define RNPGBE_RX_BASE_ADDR_HI 0x30 +#define RNPGBE_RX_BASE_ADDR_LO 0x34 +#define RNPGBE_RX_LEN 0x38 +#define RNPGBE_RX_HEAD 0x3c +#define RNPGBE_RX_TAIL 0x40 +#define M_DEFAULT_RX_FETCH 0x100020 +#define RNPGBE_RX_FETCH 0x44 +#define M_DEFAULT_INT_TIMER_R 30 +#define RNPGBE_RX_INT_TIMER 0x48 +#define M_DEFAULT_RX_INT_PKTCNT 64 +#define RNPGBE_RX_INT_PKTCNT 0x4c +#define RNPGBE_RX_TIMEOUT_TH 0x54 +#define RNPGBE_RX_SG_UNIT_BYTES 16 +#define M_DEFAULT_SG 96 +#define RNPGBE_RX_DESC_DATA_LEN (M_DEFAULT_SG * RNPGBE_RX_SG_UNIT_BYTES) +#define RNPGBE_RX_SG_LEN 0x58 +#define RNPGBE_TX_BASE_ADDR_HI 0x60 +#define RNPGBE_TX_BASE_ADDR_LO 0x64 +#define RNPGBE_TX_LEN 0x68 +#define RNPGBE_TX_HEAD 0x6c +#define RNPGBE_TX_TAIL 0x70 +#define M_DEFAULT_TX_FETCH 0x80008 +#define RNPGBE_TX_FETCH_CTRL 0x74 +#define M_DEFAULT_INT_TIMER 100 +#define RNPGBE_TX_INT_TIMER 0x78 +#define M_DEFAULT_INT_PKTCNT 48 +#define RNPGBE_TX_INT_PKTCNT 0x7c +/* | 31:24 | .... | 15:8 | 7:0 | */ +/* | pfvfnum | | tx vector | rx vector | */ +#define RING_VECTOR(n) (0x04 * (n)) + +#define M_MAX_TXD_PWR 12 +#define M_MAX_DATA_PER_TXD (0x1 << M_MAX_TXD_PWR) +#define TXD_USE_COUNT(S) DIV_ROUND_UP((S), M_MAX_DATA_PER_TXD) +#define DESC_NEEDED (MAX_SKB_FRAGS + 4) +/* Leave two descriptors between the tail and head. */ +#define RESV_DESC_NEEDED 2 +#define RNPGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) + +/* Hardware requires this to be nonzero */ +#define M_DEFAULT_MAC_IP_LEN 20 +#define mucse_for_each_ring(pos, head)\ + for (typeof((head).ring) __pos = (head).ring;\ + __pos ? ({ pos = __pos; 1; }) : 0;\ + __pos = __pos->next) +int rnpgbe_init_interrupt_scheme(struct mucse *mucse); +void rnpgbe_clear_interrupt_scheme(struct mucse *mucse); +int rnpgbe_request_mbx_irq(struct mucse *mucse); +void rnpgbe_free_mbx_irq(struct mucse *mucse); +int rnpgbe_request_irq(struct mucse *mucse); +void rnpgbe_free_irq(struct mucse *mucse); +void rnpgbe_down(struct mucse *mucse); +int rnpgbe_up_complete(struct mucse *mucse); +int rnpgbe_configure_tx(struct mucse *mucse); +int rnpgbe_configure_rx(struct mucse *mucse); +void rnpgbe_clean_all_tx_rings(struct mucse *mucse); +int rnpgbe_setup_all_tx_resources(struct mucse *mucse); +void rnpgbe_free_all_tx_resources(struct mucse *mucse); +netdev_tx_t rnpgbe_xmit_frame_ring(struct sk_buff *skb, + struct mucse_ring *tx_ring); +void rnpgbe_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats); +int rnpgbe_setup_all_rx_resources(struct mucse *mucse); +void rnpgbe_free_all_rx_resources(struct mucse *mucse); +void rnpgbe_service_task(struct work_struct *work); +#endif diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c index 70a2b0082ba8..708a7297233e 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c @@ -2,11 +2,13 @@ /* Copyright(c) 2020 - 2025 Mucse Corporation. */ #include +#include #include #include #include "rnpgbe.h" #include "rnpgbe_hw.h" +#include "rnpgbe_lib.h" #include "rnpgbe_mbx_fw.h" static const char rnpgbe_driver_name[] = "rnpgbe"; @@ -25,6 +27,32 @@ static struct pci_device_id rnpgbe_pci_tbl[] = { { }, }; +/** + * rnpgbe_configure - Configure the hardware + * @mucse: pointer to private structure + * + * Configure Tx and Rx registers in hardware. + * + * Return: 0 on success, negative errno if hardware configuration fails + **/ +static int rnpgbe_configure(struct mucse *mucse) +{ + int err; + + err = rnpgbe_configure_tx(mucse); + if (err) + return err; + + netif_addr_lock_bh(mucse->netdev); + err = rnpgbe_set_rx_mode(mucse->netdev, &mucse->netdev->uc, + &mucse->netdev->mc); + netif_addr_unlock_bh(mucse->netdev); + if (err) + return err; + + return rnpgbe_configure_rx(mucse); +} + /** * rnpgbe_open - Called when a network interface is made active * @netdev: network interface device structure @@ -32,11 +60,54 @@ static struct pci_device_id rnpgbe_pci_tbl[] = { * The open entry point is called when a network interface is made * active by the system (IFF_UP). * - * Return: 0 + * Return: 0 on success, negative value on failure **/ static int rnpgbe_open(struct net_device *netdev) { + struct mucse *mucse = netdev_priv(netdev); + int err; + + if (test_bit(__MUCSE_AXI_FAULT, &mucse->state)) + return -EIO; + + netif_carrier_off(netdev); + err = rnpgbe_request_irq(mucse); + if (err) + return err; + + err = netif_set_real_num_queues(netdev, mucse->num_tx_queues, + mucse->num_rx_queues); + if (err) + goto err_free_irqs; + + err = rnpgbe_setup_all_tx_resources(mucse); + if (err) + goto err_free_irqs; + err = rnpgbe_setup_all_rx_resources(mucse); + if (err) + goto err_free_tx; + + err = rnpgbe_configure(mucse); + if (err) + goto err_free_rx; + err = rnpgbe_up_complete(mucse); + if (err) + goto err_down; + return 0; +err_down: + rnpgbe_down(mucse); + rnpgbe_free_all_rx_resources(mucse); + rnpgbe_free_all_tx_resources(mucse); + goto err_free_irqs; +err_free_rx: + rnpgbe_free_all_rx_resources(mucse); +err_free_tx: + rnpgbe_clean_all_tx_rings(mucse); + rnpgbe_free_all_tx_resources(mucse); +err_free_irqs: + rnpgbe_free_irq(mucse); + return err; } /** @@ -50,6 +121,13 @@ static int rnpgbe_open(struct net_device *netdev) **/ static int rnpgbe_close(struct net_device *netdev) { + struct mucse *mucse = netdev_priv(netdev); + + rnpgbe_down(mucse); + rnpgbe_free_irq(mucse); + rnpgbe_free_all_tx_resources(mucse); + rnpgbe_free_all_rx_resources(mucse); + return 0; } @@ -58,25 +136,48 @@ static int rnpgbe_close(struct net_device *netdev) * @skb: skb structure to be sent * @netdev: network interface device structure * - * Return: NETDEV_TX_OK + * Return: NETDEV_TX_OK or NETDEV_TX_BUSY when insufficient descriptors **/ static netdev_tx_t rnpgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { struct mucse *mucse = netdev_priv(netdev); + struct mucse_ring *tx_ring; - dev_kfree_skb_any(skb); - mucse->stats.tx_dropped++; + tx_ring = mucse->tx_ring[skb_get_queue_mapping(skb)]; - return NETDEV_TX_OK; + if (unlikely(skb_put_padto(skb, RNPGBE_TX_MIN_PKT_LEN))) { + atomic64_inc(&tx_ring->stats->dropped); + return NETDEV_TX_OK; + } + + return rnpgbe_xmit_frame_ring(skb, tx_ring); } static const struct net_device_ops rnpgbe_netdev_ops = { - .ndo_open = rnpgbe_open, - .ndo_stop = rnpgbe_close, - .ndo_start_xmit = rnpgbe_xmit_frame, + .ndo_open = rnpgbe_open, + .ndo_stop = rnpgbe_close, + .ndo_start_xmit = rnpgbe_xmit_frame, + .ndo_set_rx_mode_async = rnpgbe_set_rx_mode, + .ndo_get_stats64 = rnpgbe_get_stats64, }; +static void rnpgbe_sw_init(struct mucse *mucse) +{ + int i; + + mucse->tx_ring_item_count = M_DEFAULT_TXD; + mucse->rx_ring_item_count = M_DEFAULT_RXD; + mucse->tx_work_limit = M_DEFAULT_TX_WORK; + + for (i = 0; i < RNPGBE_MAX_QUEUES; i++) { + u64_stats_init(&mucse->tx_stats[i].syncp); + atomic64_set(&mucse->tx_stats[i].dropped, 0); + u64_stats_init(&mucse->rx_stats[i].syncp); + atomic64_set(&mucse->rx_stats[i].dropped, 0); + } +} + /** * rnpgbe_add_adapter - Add netdev for this pci_dev * @pdev: PCI device information structure @@ -149,11 +250,18 @@ static int rnpgbe_add_adapter(struct pci_dev *pdev, } netdev->netdev_ops = &rnpgbe_netdev_ops; + netdev->priv_flags |= IFF_UNICAST_FLT; + rnpgbe_sw_init(mucse); err = rnpgbe_reset_hw(hw); if (err) { dev_err(&pdev->dev, "Hw reset failed %d\n", err); goto err_powerdown; } + err = rnpgbe_setup_default_link(hw); + if (err) { + dev_err(&pdev->dev, "Setup link failed %d\n", err); + goto err_powerdown; + } err = rnpgbe_get_permanent_mac(hw, perm_addr); if (!err) { @@ -166,11 +274,46 @@ static int rnpgbe_add_adapter(struct pci_dev *pdev, goto err_powerdown; } + INIT_DELAYED_WORK(&mucse->serv_task, rnpgbe_service_task); + spin_lock_init(&mucse->link_lock); + mucse->link_event_ready = false; + mucse->link_pending = false; + + err = rnpgbe_init_interrupt_scheme(mucse); + if (err) { + dev_err(&pdev->dev, "init interrupt failed %d\n", err); + goto err_powerdown; + } + + err = netif_set_real_num_queues(netdev, mucse->num_tx_queues, + mucse->num_rx_queues); + if (err) + goto err_clear_interrupt; + + err = rnpgbe_request_mbx_irq(mucse); + if (err) { + dev_err(&pdev->dev, "register mbx irq failed %d\n", err); + goto err_clear_interrupt; + } + + netdev->features |= NETIF_F_SG; + netdev->hw_features |= NETIF_F_SG; + if (dma_get_mask(&pdev->dev) > DMA_BIT_MASK(32)) { + netdev->features |= NETIF_F_HIGHDMA; + netdev->hw_features |= NETIF_F_HIGHDMA; + } + + netif_carrier_off(netdev); err = register_netdev(netdev); if (err) - goto err_powerdown; + goto err_remove_mbx; return 0; + +err_remove_mbx: + rnpgbe_free_mbx_irq(mucse); +err_clear_interrupt: + rnpgbe_clear_interrupt_scheme(mucse); err_powerdown: /* notify powerdown only powerup ok */ if (!err_notify) { @@ -203,12 +346,7 @@ static int rnpgbe_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) return err; - err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(56)); - if (err) { - dev_err(&pdev->dev, - "No usable DMA configuration, aborting %d\n", err); - goto err_disable_dev; - } + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(56)); err = pci_request_mem_regions(pdev, rnpgbe_driver_name); if (err) { @@ -236,6 +374,26 @@ static int rnpgbe_probe(struct pci_dev *pdev, const struct pci_device_id *id) return err; } +/** + * rnpgbe_notify_powerdown - Notify firmware that the driver is going down + * @mucse: pointer to private structure + * + * Skip the mailbox access if an earlier shutdown has already disabled the + * PCI device. + **/ +static void rnpgbe_notify_powerdown(struct mucse *mucse) +{ + int err; + + if (!pci_is_enabled(mucse->pdev)) + return; + + err = rnpgbe_send_notify(&mucse->hw, false, mucse_fw_powerup); + if (err) + dev_warn(&mucse->pdev->dev, + "Send powerdown to hw failed %d\n", err); +} + /** * rnpgbe_rm_adapter - Remove netdev for this mucse structure * @pdev: PCI device information struct @@ -245,17 +403,15 @@ static int rnpgbe_probe(struct pci_dev *pdev, const struct pci_device_id *id) static void rnpgbe_rm_adapter(struct pci_dev *pdev) { struct mucse *mucse = pci_get_drvdata(pdev); - struct mucse_hw *hw = &mucse->hw; struct net_device *netdev; - int err; if (!mucse) return; netdev = mucse->netdev; unregister_netdev(netdev); - err = rnpgbe_send_notify(hw, false, mucse_fw_powerup); - if (err) - dev_warn(&pdev->dev, "Send powerdown to hw failed %d\n", err); + rnpgbe_free_mbx_irq(mucse); + rnpgbe_notify_powerdown(mucse); + rnpgbe_clear_interrupt_scheme(mucse); free_netdev(netdev); } @@ -272,7 +428,8 @@ static void rnpgbe_remove(struct pci_dev *pdev) { rnpgbe_rm_adapter(pdev); pci_release_mem_regions(pdev); - pci_disable_device(pdev); + if (pci_is_enabled(pdev)) + pci_disable_device(pdev); } /** @@ -287,9 +444,14 @@ static void rnpgbe_dev_shutdown(struct pci_dev *pdev) rtnl_lock(); netif_device_detach(netdev); if (netif_running(netdev)) - rnpgbe_close(netdev); + dev_close(netdev); rtnl_unlock(); - pci_disable_device(pdev); + + rnpgbe_free_mbx_irq(mucse); + rnpgbe_notify_powerdown(mucse); + rnpgbe_clear_interrupt_scheme(mucse); + if (pci_is_enabled(pdev)) + pci_disable_device(pdev); } /** diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.c index c46408698263..ef00b948ad0f 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.c +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.c @@ -84,7 +84,7 @@ static u32 mucse_mbx_get_lock_pf(struct mucse_hw *hw) * @hw: pointer to the HW structure * * Pair with mucse_release_mbx_lock_pf() - * This function maybe used in an irq handler. + * All mailbox access runs in process context. * * Return: 0 on success, negative errno on failure **/ @@ -93,11 +93,11 @@ static int mucse_obtain_mbx_lock_pf(struct mucse_hw *hw) struct mucse_mbx_info *mbx = &hw->mbx; u32 val; - return read_poll_timeout_atomic(mucse_mbx_get_lock_pf, - val, val & MUCSE_MBX_PFU, - mbx->delay_us, - mbx->timeout_us, - false, hw); + return read_poll_timeout(mucse_mbx_get_lock_pf, + val, val & MUCSE_MBX_PFU, + mbx->delay_us, + mbx->timeout_us, + false, hw); } /** @@ -197,7 +197,8 @@ static int mucse_read_mbx_pf(struct mucse_hw *hw, __le32 *msg, u16 size) * mucse_check_for_msg_pf - Check to see if the fw has sent mail * @hw: pointer to the HW structure * - * Return: 0 if the fw has set the Status bit or else -EIO + * Return: 0 if firmware has posted a new message, -ENOMSG if there is no + * new message, or -EIO if the mailbox has been reset **/ static int mucse_check_for_msg_pf(struct mucse_hw *hw) { @@ -209,8 +210,13 @@ static int mucse_check_for_msg_pf(struct mucse_hw *hw) * mbx command. Return -EIO if in this state, others * fw == hw->mbx.fw_req means no new msg. **/ - if (fw_req == 0 || fw_req == hw->mbx.fw_req) + /* Firmware skips zero for valid FW-to-PF requests and wraps the + * counter from U16_MAX to 1. + **/ + if (!fw_req) return -EIO; + if (fw_req == hw->mbx.fw_req) + return -ENOMSG; return 0; } @@ -252,6 +258,68 @@ int mucse_poll_and_read_mbx(struct mucse_hw *hw, __le32 *msg, u16 size) return mucse_read_mbx_pf(hw, msg, size); } +/** + * mucse_mbx_event_begin - Lock and read a pending firmware event + * @hw: pointer to the HW structure + * @msg: the message buffer + * @size: length of buffer + * + * On success the hardware mailbox remains locked. The caller must process + * the event and call mucse_mbx_event_end() to acknowledge it and release + * the hardware mailbox lock. + * + * Firmware releases the hardware mailbox lock before raising the event + * interrupt and does not access the shared window again until the PF + * acknowledges the event. Therefore, failure to acquire the lock indicates + * a mailbox fault rather than transient contention. + * + * Return: 0 on success, -ENOMSG if no event is pending, or another negative + * errno on failure + **/ +int mucse_mbx_event_begin(struct mucse_hw *hw, __le32 *msg, u16 size) +{ + const int size_in_words = size / sizeof(__le32); + struct mucse_mbx_info *mbx = &hw->mbx; + int off = MUCSE_MBX_FWPF_SHM; + int err; + + err = mucse_check_for_msg_pf(hw); + if (err) + return err; + + err = mucse_obtain_mbx_lock_pf(hw); + if (err) + return err; + + /* Check again after taking ownership of the shared mailbox. */ + err = mucse_check_for_msg_pf(hw); + if (err) { + mucse_release_mbx_lock_pf(hw, false); + return err; + } + + for (int i = 0; i < size_in_words; i++) + msg[i] = cpu_to_le32(mbx_data_rd32(mbx, off + 4 * i)); + + return 0; +} + +/** + * mucse_mbx_event_end - Acknowledge a firmware event and unlock the mailbox + * @hw: pointer to the HW structure + * + * Pair with a successful mucse_mbx_event_begin(). + **/ +void mucse_mbx_event_end(struct mucse_hw *hw) +{ + struct mucse_mbx_info *mbx = &hw->mbx; + + mbx_data_wr32(mbx, MUCSE_MBX_FWPF_SHM, 0); + hw->mbx.fw_req = mucse_mbx_get_fwreq(mbx); + mucse_mbx_inc_pf_ack(hw); + mucse_release_mbx_lock_pf(hw, false); +} + /** * mucse_mbx_get_fwack - Read fw ack from reg * @mbx: pointer to the MBX structure @@ -286,6 +354,36 @@ static void mucse_mbx_inc_pf_req(struct mucse_hw *hw) mbx_data_wr32(mbx, MUCSE_MBX_PF2FW_CNT, val); } +/** + * mucse_ack_pending_event - Acknowledge a level-like firmware event + * @hw: pointer to the HW structure + * + * The caller must own the hardware mailbox lock. + * + * Return: 0 on success, negative errno on failure + **/ +static int mucse_ack_pending_event(struct mucse_hw *hw) +{ + struct mucse_mbx_info *mbx = &hw->mbx; + u16 fw_req; + + fw_req = mucse_mbx_get_fwreq(mbx); + if (fw_req == hw->mbx.fw_req) + return 0; + if (!fw_req) + return -EIO; + + /* Firmware only posts LINK_CHANGE_EVT asynchronously. It is level-like, + * so leaving the link snapshot unchanged lets firmware report the + * current state again if reporting stays enabled. + */ + mbx_data_wr32(mbx, MUCSE_MBX_FWPF_SHM, 0); + hw->mbx.fw_req = fw_req; + mucse_mbx_inc_pf_ack(hw); + + return 0; +} + /** * mucse_write_mbx_pf - Place a message in the mailbox * @hw: pointer to the HW structure @@ -320,6 +418,44 @@ static int mucse_write_mbx_pf(struct mucse_hw *hw, const __le32 *msg, u16 size) return 0; } +/** + * mucse_write_mbx_pf_coalesce_event - Write around a level-like event + * @hw: pointer to the HW structure + * @msg: the message buffer + * @size: length of buffer + * + * Return: 0 on success, negative errno on failure + **/ +static int mucse_write_mbx_pf_coalesce_event(struct mucse_hw *hw, + const __le32 *msg, u16 size) +{ + const int size_in_words = size / sizeof(__le32); + struct mucse_mbx_info *mbx = &hw->mbx; + int err; + + err = mucse_obtain_mbx_lock_pf(hw); + if (err) + return err; + + err = mucse_ack_pending_event(hw); + if (err) + goto release; + + for (int i = 0; i < size_in_words; i++) + mbx_data_wr32(mbx, MUCSE_MBX_FWPF_SHM + i * 4, + le32_to_cpu(msg[i])); + + hw->mbx.fw_ack = mucse_mbx_get_fwack(mbx); + mucse_mbx_inc_pf_req(hw); + mucse_release_mbx_lock_pf(hw, true); + + return 0; + +release: + mucse_release_mbx_lock_pf(hw, false); + return err; +} + /** * mucse_check_for_ack_pf - Check to see if the fw has ACKed * @hw: pointer to the HW structure @@ -381,6 +517,33 @@ int mucse_write_and_wait_ack_mbx(struct mucse_hw *hw, const __le32 *msg, return mucse_poll_for_ack(hw); } +/** + * mucse_write_mbx_coalesce_event - Send a command while coalescing an event + * @hw: pointer to the HW structure + * @msg: the message buffer + * @size: length of buffer + * + * A pending link event is acknowledged without updating its snapshot, then + * the PF command is written while the hardware mailbox remains locked. If + * event reporting remains enabled, firmware reports the event again because + * its snapshot still differs from the hardware state. This function waits + * for firmware to acknowledge the PF command before returning. + * + * Return: 0 after firmware acknowledges the command, negative errno on + * failure + **/ +int mucse_write_mbx_coalesce_event(struct mucse_hw *hw, + const __le32 *msg, u16 size) +{ + int err; + + err = mucse_write_mbx_pf_coalesce_event(hw, msg, size); + if (err) + return err; + + return mucse_poll_for_ack(hw); +} + /** * mucse_mbx_reset - Reset mbx info, sync info from regs * @hw: pointer to the HW structure diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.h b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.h index 75b88b18b04d..50597dd9dde9 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.h +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx.h @@ -16,6 +16,10 @@ int mucse_write_and_wait_ack_mbx(struct mucse_hw *hw, const __le32 *msg, u16 size); +int mucse_write_mbx_coalesce_event(struct mucse_hw *hw, + const __le32 *msg, u16 size); void mucse_init_mbx_params_pf(struct mucse_hw *hw); int mucse_poll_and_read_mbx(struct mucse_hw *hw, __le32 *msg, u16 size); +int mucse_mbx_event_begin(struct mucse_hw *hw, __le32 *msg, u16 size); +void mucse_mbx_event_end(struct mucse_hw *hw); #endif /* _RNPGBE_MBX_H */ diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.c index 5ba74997beac..683e0ea74113 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.c +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.c @@ -3,6 +3,7 @@ #include #include +#include #include "rnpgbe.h" #include "rnpgbe_mbx.h" @@ -197,3 +198,285 @@ int mucse_mbx_get_macaddr(struct mucse_hw *hw, int pfvfnum, return 0; } + +/** + * mucse_mbx_set_link - Configure firmware link settings + * @hw: pointer to the HW structure + * @advertising: firmware link-mode advertisement mask + * @autoneg: whether to enable autonegotiation + * @speed: forced link speed when autonegotiation is disabled + * @duplex: forced duplex mode when autonegotiation is disabled + * @mdix_ctrl: MDI/MDI-X control mode + * + * Firmware ignores @speed, @duplex when @autoneg is true. + * + * Return: 0 on success, negative errno on failure + **/ +int mucse_mbx_set_link(struct mucse_hw *hw, u32 advertising, bool autoneg, + u32 speed, u32 duplex, u32 mdix_ctrl) +{ + union mbx_fw_cmd_req_u req = { + .r = { + .datalen = cpu_to_le16(sizeof(req.r.phy_link_set) + + MUCSE_MBX_REQ_HDR_LEN), + .opcode = cpu_to_le16(PHY_LINK_SET), + .phy_link_set = { + .adv_speed_mask = cpu_to_le32(advertising), + .autoneg = cpu_to_le32(autoneg), + .speed = cpu_to_le32(speed), + .duplex = cpu_to_le32(duplex), + .nr_lane = cpu_to_le32(hw->port), + .tp_mdix_ctrl = cpu_to_le32(mdix_ctrl), + }, + }, + }; + int len, err; + + len = le16_to_cpu(req.r.datalen); + mutex_lock(&hw->mbx.lock); + err = mucse_write_and_wait_ack_mbx(hw, req.dwords, len); + mutex_unlock(&hw->mbx.lock); + + return err; +} + +/** + * mucse_mbx_phyup - Request that firmware bring the PHY up or down + * @hw: pointer to the HW structure + * @is_phyup: true for up, false for down + * + * mucse_mbx_phyup echo fw to change phy status + * + * Return: 0 on success, negative errno on failure + **/ +int mucse_mbx_phyup(struct mucse_hw *hw, bool is_phyup) +{ + union mbx_fw_cmd_req_u req = { + .r = { + .datalen = cpu_to_le16(sizeof(req.r.phy_status) + + MUCSE_MBX_REQ_HDR_LEN), + .opcode = cpu_to_le16(SET_PHY_UP), + .phy_status = { + .port_mask = cpu_to_le32(BIT(hw->port)), + .status = cpu_to_le32(is_phyup ? 1 : 0), + }, + }, + }; + int len, err; + + len = le16_to_cpu(req.r.datalen); + mutex_lock(&hw->mbx.lock); + err = mucse_write_mbx_coalesce_event(hw, req.dwords, len); + mutex_unlock(&hw->mbx.lock); + + return err; +} + +/** + * mucse_mbx_link_report - Configure firmware link-change event reporting + * @hw: pointer to the HW structure + * @is_report: true for report, false for no + * + * mucse_mbx_link_report echo fw to change event report state + * + * Return: 0 on success, negative errno on failure + **/ +int mucse_mbx_link_report(struct mucse_hw *hw, bool is_report) +{ + union mbx_fw_cmd_req_u req = { + .r = { + .datalen = cpu_to_le16(sizeof(req.r.report_status) + + MUCSE_MBX_REQ_HDR_LEN), + .opcode = cpu_to_le16(LINK_REPORT_EN), + .report_status = { + .port_mask = cpu_to_le16(BIT(hw->port)), + .status = cpu_to_le16(is_report ? 1 : 0), + }, + }, + }; + int len, err; + + len = le16_to_cpu(req.r.datalen); + mutex_lock(&hw->mbx.lock); + err = mucse_write_mbx_coalesce_event(hw, req.dwords, len); + mutex_unlock(&hw->mbx.lock); + + return err; +} + +static bool mucse_link_speed_valid(const struct mbx_fw_cmd_req *req) +{ + switch (le16_to_cpu(req->link_stat.st.speed)) { + case 10: + case 100: + case 1000: + return true; + default: + return false; + } +} + +static bool mucse_link_is_up(const struct mucse_hw *hw, + const struct mbx_fw_cmd_req *req) +{ + return le16_to_cpu(req->link_stat.port_status) & BIT(hw->port); +} + +/** + * mucse_update_link_status_reg - update driver speed inf to reg + * @hw: pointer to the HW structure + * @req: pointer to req data + * + * Update the driver's link-state snapshot exported to firmware. Firmware + * sends a new event when this snapshot differs from the hardware state. + * The default snapshot clears the driver-reported fields; + * a valid event then repopulates them, including the LLDP status in bit 6. + * + **/ +static void mucse_update_link_status_reg(struct mucse_hw *hw, + struct mbx_fw_cmd_req *req) +{ + u16 status = le16_to_cpu(req->link_stat.st.status); + u16 speed = le16_to_cpu(req->link_stat.st.speed); + u32 value; + + value = M_DEFAULT_ST; + + if (mucse_link_is_up(hw, req)) { + value |= BIT(0); + switch (speed) { + case 10: + value |= (mucse_speed_10 << 8); + break; + case 100: + value |= (mucse_speed_100 << 8); + break; + case 1000: + value |= (mucse_speed_1000 << 8); + break; + default: + break; + } + + value |= FIELD_PREP(BIT(4), + !!(req->link_stat.st.flags & DUPLEX_BIT)); + value |= FIELD_PREP(GENMASK_U32(25, 24), + status & GENMASK(1, 0)); + } else { + value &= ~BIT(0); + } + + if (status & ST_STATUS_LLDP_STATUS_MASK) + value |= BIT(6); + else + value &= ~BIT(6); + + mucse_hw_wr32(hw, RNPGBE_LINK_ST, value); +} + +/** + * mucse_mbx_fw_req_handler - Handle fw req + * @hw: pointer to the HW structure + * @req: pointer to req data + * + * mucse_mbx_fw_req_handler handler fw req, such as a link event req. + **/ +static void mucse_mbx_fw_req_handler(struct mucse_hw *hw, + struct mbx_fw_cmd_req *req) +{ + struct mucse *mucse = container_of(hw, struct mucse, hw); + u32 magic = le32_to_cpu(req->link_stat.port_magic); + unsigned long flags; + + if (le16_to_cpu(req->opcode) == LINK_CHANGE_EVT) { + u16 speed = le16_to_cpu(req->link_stat.st.speed); + + spin_lock_irqsave(&mucse->link_lock, flags); + if (magic != ST_VALID_MAGIC) { + /* Do not change the cached state for an invalid event. + * Use an invalid speed encoding to make firmware report + * again. + */ + mucse_hw_wr32(hw, RNPGBE_LINK_ST, M_INVALID_ST); + spin_unlock_irqrestore(&mucse->link_lock, flags); + return; + } + + if (mucse_link_is_up(hw, req) && + !mucse_link_speed_valid(req)) { + /* Do not acknowledge an invalid speed as valid. + * Keep the snapshot mismatched so firmware retries it. + * Firmware limits link reports to one per 500 ms. + */ + mucse_hw_wr32(hw, RNPGBE_LINK_ST, M_INVALID_ST); + spin_unlock_irqrestore(&mucse->link_lock, flags); + dev_warn_ratelimited(&hw->pdev->dev, + "unsupported link speed %u Mbps\n", + speed); + return; + } + + if (!mucse->link_event_ready) { + rnpgbe_set_link(hw, false); + mucse_update_link_status_reg(hw, req); + spin_unlock_irqrestore(&mucse->link_lock, flags); + return; + } + + if (mucse_link_is_up(hw, req)) + hw->link = true; + else + hw->link = false; + + hw->speed = le16_to_cpu(req->link_stat.st.speed); + hw->duplex = req->link_stat.st.flags & DUPLEX_BIT; + rnpgbe_set_link(hw, mucse_link_is_up(hw, req)); + /* update regs to notify link info is received */ + mucse_update_link_status_reg(hw, req); + mucse->link_pending = true; + /* Run link handling immediately. */ + mod_delayed_work(system_percpu_wq, &mucse->serv_task, 0); + spin_unlock_irqrestore(&mucse->link_lock, flags); + } +} + +/** + * mucse_fw_handle_event - Handle one pending firmware event + * @hw: pointer to the hardware structure + **/ +static void mucse_fw_handle_event(struct mucse_hw *hw) +{ + union mbx_fw_cmd_req_u msg = {}; + int err; + + /* try to check and read fw req */ + mutex_lock(&hw->mbx.lock); + err = mucse_mbx_event_begin(hw, msg.dwords, sizeof(msg)); + /* A PF command may have coalesced the event before stale mailbox work + * runs, in which case there is no event left to handle. + */ + if (err) { + mutex_unlock(&hw->mbx.lock); + if (err != -ENOMSG) + dev_warn_ratelimited(&hw->pdev->dev, + "failed to read firmware event: %d\n", + err); + return; + } + + mucse_mbx_fw_req_handler(hw, &msg.r); + mucse_mbx_event_end(hw); + mutex_unlock(&hw->mbx.lock); +} + +/** + * mucse_fw_irq_handler - Handle one pending firmware mailbox event + * @hw: pointer to the HW structure + * + * Process at most one event per work-item invocation. The caller requeues + * mailbox work when a dedicated mailbox interrupt arrives during handling. + **/ +void mucse_fw_irq_handler(struct mucse_hw *hw) +{ + mucse_fw_handle_event(hw); +} diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.h b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.h index fe996aeffc4d..93ce0e796442 100644 --- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.h +++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_mbx_fw.h @@ -14,9 +14,19 @@ enum MUCSE_FW_CMD { GET_HW_INFO = 0x0601, GET_MAC_ADDRESS = 0x0602, RESET_HW = 0x0603, + LINK_CHANGE_EVT = 0x0608, + LINK_REPORT_EN = 0x0613, + PHY_LINK_SET = 0x0630, + SET_PHY_UP = 0x0800, POWER_UP = 0x0803, }; +#define RNPGBE_FW_ADV_10_FULL BIT(2) +#define RNPGBE_FW_ADV_100_FULL BIT(3) +#define RNPGBE_FW_ADV_1000_FULL BIT(4) +#define RNPGBE_FW_ADV_10_HALF BIT(10) +#define RNPGBE_FW_ADV_100_HALF BIT(11) + struct mucse_hw_info { u8 link_stat; u8 port_mask; @@ -36,6 +46,16 @@ struct mucse_hw_info { __le32 ext_info; } __packed; +#define ST_STATUS_LLDP_STATUS_MASK BIT(12) + +#define DUPLEX_BIT BIT(0) +struct st_status { + u8 phyid; + u8 flags; + __le16 speed; + __le16 status; +} __packed; + struct mbx_fw_cmd_req { __le16 flags; __le16 opcode; @@ -55,6 +75,32 @@ struct mbx_fw_cmd_req { __le32 port_mask; __le32 pfvf_num; } get_mac_addr; + struct { + __le32 port_mask; + __le32 status; + } phy_status; + struct { + __le32 adv_speed_mask; + __le32 autoneg; + __le32 speed; + __le32 duplex; + __le32 nr_lane; + __le32 tp_mdix_ctrl; + } phy_link_set; + struct { + /* LINK_REPORT_EN uses its own 16-bit payload layout: + * status precedes port_mask. + */ + __le16 status; + __le16 port_mask; + } report_status; + struct { + __le16 changed_lanes; + __le16 port_status; + __le32 port_magic; +#define ST_VALID_MAGIC 0xa4a6a8a9 + struct st_status st; + } link_stat; }; } __packed; @@ -94,9 +140,23 @@ union mbx_fw_cmd_reply_u { __le32 dwords[sizeof(struct mbx_fw_cmd_reply) / sizeof(__le32)]; }; +/* Firmware encoding for RNPGBE_LINK_ST[11:8]. Valid link events report + * only these three speeds. + */ +enum mucse_speed { + mucse_speed_10 = 0, + mucse_speed_100 = 1, + mucse_speed_1000 = 2, +}; + int mucse_mbx_sync_fw(struct mucse_hw *hw); int mucse_mbx_powerup(struct mucse_hw *hw, bool is_powerup); int mucse_mbx_reset_hw(struct mucse_hw *hw); int mucse_mbx_get_macaddr(struct mucse_hw *hw, int pfvfnum, u8 *mac_addr, int port); +int mucse_mbx_set_link(struct mucse_hw *hw, u32 advertising, bool autoneg, + u32 speed, u32 duplex, u32 mdix_ctrl); +int mucse_mbx_phyup(struct mucse_hw *hw, bool is_phyup); +int mucse_mbx_link_report(struct mucse_hw *hw, bool is_report); +void mucse_fw_irq_handler(struct mucse_hw *hw); #endif /* _RNPGBE_MBX_FW_H */