diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 43c46d483eb8..88354e7570f0 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -77,6 +78,22 @@ #define R8169_RX_RING_BYTES (NUM_RX_DESC * sizeof(struct RxDesc)) #define R8169_TX_STOP_THRS (MAX_SKB_FRAGS + 1) #define R8169_TX_START_THRS (2 * R8169_TX_STOP_THRS) +#define R8169_MAX_RX_QUEUES 8 +#define R8169_DEFAULT_RX_QUEUES 1 +#define R8169_MAX_TX_QUEUES 1 +#define R8127_MAX_NUM_IRQVEC 32 +#define R8127_MIN_NUM_IRQVEC 30 +#define R8169_IRQ_DEFAULT 1 +#define RTL_RSS_KEY_SIZE 40 +#define RSS_CPU_NUM_MASK GENMASK(18, 16) +#define RSS_HASH_MASK GENMASK(10, 8) +#define RTL_MAX_INDIRECTION_TABLE_ENTRIES 128 +#define RXS_RSS_UDP BIT(27) +#define RXS_RSS_IPV4 BIT(28) +#define RXS_RSS_IPV6 BIT(29) +#define RXS_RSS_TCP BIT(30) +#define RXS_RSS_L3_TYPE_MASK (RXS_RSS_IPV4 | RXS_RSS_IPV6) +#define RXS_RSS_L4_TYPE_MASK (RXS_RSS_TCP | RXS_RSS_UDP) #define OCP_STD_PHY_BASE 0xa400 @@ -463,11 +480,16 @@ enum rtl8125_registers { TxPoll_8125 = 0x90, LEDSEL3 = 0x96, MAC0_BKP = 0x19e0, + RDSAR_Q1_LOW = 0x4000, RSS_CTRL_8125 = 0x4500, Q_NUM_CTRL_8125 = 0x4800, EEE_TXIDLE_TIMER_8125 = 0x6048, + IMR_CLEAR_VEC_MAP_REG = 0x0d00, + ISR_VEC_MAP_REG = 0x0d04, + IMR_SET_VEC_MAP_REG = 0x0d0c, }; +#define MSIX_ID_VEC_MAP_LINKCHG 29 #define LEDSEL_MASK_8125 0x23f #define RX_VLAN_INNER_8125 BIT(22) @@ -496,6 +518,9 @@ enum rtl_register_content { RxRUNT = (1 << 20), RxCRC = (1 << 19), + RXRUNT_RSS = (1 << 21), + RXCRC_RSS = (1 << 20), + /* ChipCmdBits */ StopReq = 0x80, CmdReset = 0x10, @@ -598,6 +623,23 @@ enum rtl_register_content { /* magic enable v2 */ MagicPacket_v2 = (1 << 16), /* Wake up when receives a Magic Packet */ +#define ISRIMR_LINKCHG BIT(29) +#define ISRIMR_TOK_Q0 BIT(8) +#define ISRIMR_ROK_Q0 BIT(0) +#define RTL_DESC_TYPE_CTRL 0xd8 +#define RSS_KEY_REG 0x4600 +#define RSS_INDIRECTION_TBL_REG 0x4700 +#define RSS_CTRL_TCP_IPV4_SUPP BIT(0) +#define RTL_DESC_TYPE_RSS BIT(1) +#define RSS_CTRL_IPV4_SUPP BIT(1) +#define RSS_CTRL_TCP_IPV6_SUPP BIT(2) +#define RSS_CTRL_IPV6_SUPP BIT(3) +#define RSS_CTRL_IPV6_EXT_SUPP BIT(4) +#define RSS_CTRL_TCP_IPV6_EXT_SUPP BIT(5) +#define RSS_CTRL_UDP_IPV4_SUPP BIT(11) +#define RSS_CTRL_UDP_IPV6_SUPP BIT(12) +#define RX_RES_RSS BIT(22) +#define RTL_RX_Q_NUM_MASK GENMASK(4, 2) }; enum rtl_desc_bit { @@ -655,6 +697,11 @@ enum rtl_rx_desc_bit { #define RxProtoIP (PID1 | PID0) #define RxProtoMask RxProtoIP +#define RX_UDPT_DESC_RSS BIT(19) +#define RX_TCPT_DESC_RSS BIT(18) +#define RX_UDPF_DESC_RSS BIT(16) /* UDP/IP checksum failed */ +#define RX_TCPF_DESC_RSS BIT(15) /* TCP/IP checksum failed */ + IPFail = (1 << 16), /* IP checksum failed */ UDPFail = (1 << 15), /* UDP/IP checksum failed */ TCPFail = (1 << 14), /* TCP/IP checksum failed */ @@ -676,9 +723,27 @@ struct TxDesc { }; struct RxDesc { - __le32 opts1; - __le32 opts2; - __le64 addr; + union { + /* RX_DESC_TYPE_DEFAULT */ + struct { + __le32 opts1; + __le32 opts2; + __le64 addr; + }; + + /* RX_DESC_TYPE_RSS */ + struct { + union { + __le64 rss_addr; + struct { + __le32 rss_info; + __le32 rss_result; + } rss_dword; + }; + __le32 rss_opts2; + __le32 rss_opts1; + }; + }; }; struct ring_info { @@ -750,34 +815,63 @@ enum rtl_dash_type { RTL_DASH_25_BP, }; +enum rx_desc_type { + RX_DESC_TYPE_DEFAULT, + RX_DESC_TYPE_RSS, +}; + +struct rtl8169_rx_ring { + u32 cur_rx; + struct RxDesc *rx_desc_array; + dma_addr_t rx_buf_dma_addr[NUM_RX_DESC]; + dma_addr_t rx_phy_addr; + struct page *rx_databuff[NUM_RX_DESC]; + + struct { + u64 rx_errors; + u64 rx_dropped; + u64 rx_length_errors; + u64 rx_crc_errors; + u64 multicast; + struct u64_stats_sync syncp; + } stats; +}; + enum rtl_sfp_mode { RTL_SFP_NONE, RTL_SFP_8168_AF, RTL_SFP_8127_ATF, }; +struct rtl8169_rss_data { + u8 rss_key[RTL_RSS_KEY_SIZE]; + u8 rss_indir_tbl[RTL_MAX_INDIRECTION_TABLE_ENTRIES]; + unsigned int hw_supp_indir_tbl_entries; +}; + struct rtl8169_private { void __iomem *mmio_addr; /* memory map physical address */ struct pci_dev *pci_dev; struct net_device *dev; struct phy_device *phydev; - struct napi_struct napi; enum mac_version mac_version; enum rtl_dash_type dash_type; enum rtl_sfp_mode sfp_mode; - u32 cur_rx; /* Index into the Rx descriptor buffer of next Rx pkt. */ u32 cur_tx; /* Index into the Tx descriptor buffer of next Rx pkt. */ u32 dirty_tx; struct TxDesc *TxDescArray; /* 256-aligned Tx descriptor ring */ - struct RxDesc *RxDescArray; /* 256-aligned Rx descriptor ring */ dma_addr_t TxPhyAddr; - dma_addr_t RxPhyAddr; - struct page *Rx_databuff[NUM_RX_DESC]; /* Rx data buffers */ struct ring_info tx_skb[NUM_TX_DESC]; /* Tx data buffers */ + struct napi_struct *rtl8169_napi; + struct rtl8169_rx_ring *rx_ring; + unsigned int num_rx_rings; u16 cp_cmd; u16 tx_lpi_timer; u32 irq_mask; - int irq; + unsigned int hw_supp_num_rx_queues; + struct rtl8169_rss_data *rss_data; + unsigned int irq_nvecs; + enum rx_desc_type init_rx_desc_type; struct clk *clk; int speed; @@ -1692,6 +1786,11 @@ static enum rtl_sfp_mode rtl_get_sfp_mode(struct rtl8169_private *tp) return RTL_SFP_NONE; } +static bool rtl_hw_support_rss(struct rtl8169_private *tp) +{ + return tp->mac_version == RTL_GIGA_MAC_VER_80; +} + static enum rtl_dash_type rtl_get_dash_type(struct rtl8169_private *tp) { switch (tp->mac_version) { @@ -1758,26 +1857,38 @@ static u32 rtl_get_events(struct rtl8169_private *tp) static void rtl_ack_events(struct rtl8169_private *tp, u32 bits) { - if (rtl_is_8125(tp)) - RTL_W32(tp, IntrStatus_8125, bits); - else + if (rtl_is_8125(tp)) { + if (tp->irq_nvecs > 1) + RTL_W32(tp, ISR_VEC_MAP_REG, bits); + else + RTL_W32(tp, IntrStatus_8125, bits); + } else { RTL_W16(tp, IntrStatus, bits); + } } static void rtl_irq_disable(struct rtl8169_private *tp) { - if (rtl_is_8125(tp)) - RTL_W32(tp, IntrMask_8125, 0); - else + if (rtl_is_8125(tp)) { + if (tp->irq_nvecs > 1) + RTL_W32(tp, IMR_CLEAR_VEC_MAP_REG, 0xffffffff); + else + RTL_W32(tp, IntrMask_8125, 0); + } else { RTL_W16(tp, IntrMask, 0); + } } static void rtl_irq_enable(struct rtl8169_private *tp) { - if (rtl_is_8125(tp)) - RTL_W32(tp, IntrMask_8125, tp->irq_mask); - else + if (rtl_is_8125(tp)) { + if (tp->irq_nvecs > 1) + RTL_W32(tp, IMR_SET_VEC_MAP_REG, tp->irq_mask); + else + RTL_W32(tp, IntrMask_8125, tp->irq_mask); + } else { RTL_W16(tp, IntrMask, tp->irq_mask); + } } static void rtl8169_irq_mask_and_ack(struct rtl8169_private *tp) @@ -1979,9 +2090,20 @@ static inline u32 rtl8169_tx_vlan_tag(struct sk_buff *skb) TxVlanTag | swab16(skb_vlan_tag_get(skb)) : 0x00; } -static void rtl8169_rx_vlan_tag(struct RxDesc *desc, struct sk_buff *skb) +static void rtl8169_rx_vlan_tag(struct rtl8169_private *tp, + struct RxDesc *desc, + struct sk_buff *skb) { - u32 opts2 = le32_to_cpu(desc->opts2); + u32 opts2; + + switch (tp->init_rx_desc_type) { + case RX_DESC_TYPE_RSS: + opts2 = le32_to_cpu(desc->rss_opts2); + break; + default: + opts2 = le32_to_cpu(desc->opts2); + break; + } if (opts2 & RxVlanTag) __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), swab16(opts2 & 0xffff)); @@ -2514,34 +2636,6 @@ static int rtl8169_nway_reset(struct net_device *dev) return phylink_ethtool_nway_reset(tp->phylink); } -static const struct ethtool_ops rtl8169_ethtool_ops = { - .supported_coalesce_params = ETHTOOL_COALESCE_USECS | - ETHTOOL_COALESCE_MAX_FRAMES, - .get_drvinfo = rtl8169_get_drvinfo, - .get_regs_len = rtl8169_get_regs_len, - .get_link = ethtool_op_get_link, - .get_coalesce = rtl_get_coalesce, - .set_coalesce = rtl_set_coalesce, - .get_regs = rtl8169_get_regs, - .get_wol = rtl8169_get_wol, - .set_wol = rtl8169_set_wol, - .get_strings = rtl8169_get_strings, - .get_sset_count = rtl8169_get_sset_count, - .get_ethtool_stats = rtl8169_get_ethtool_stats, - .get_ts_info = ethtool_op_get_ts_info, - .nway_reset = rtl8169_nway_reset, - .get_eee = rtl8169_get_eee, - .set_eee = rtl8169_set_eee, - .get_link_ksettings = rtl8169_get_link_ksettings, - .set_link_ksettings = rtl8169_set_link_ksettings, - .get_ringparam = rtl8169_get_ringparam, - .get_pause_stats = rtl8169_get_pause_stats, - .get_pauseparam = rtl8169_get_pauseparam, - .set_pauseparam = rtl8169_set_pauseparam, - .get_eth_mac_stats = rtl8169_get_eth_mac_stats, - .get_eth_ctrl_stats = rtl8169_get_eth_ctrl_stats, -}; - static const struct rtl_chip_info *rtl8169_get_chip_version(u32 xid, bool gmii) { /* Chips combining a 1Gbps MAC with a 100Mbps PHY */ @@ -2717,9 +2811,25 @@ static void rtl_init_rxcfg(struct rtl8169_private *tp) } } +static void rtl8169_rx_desc_init(struct rtl8169_private *tp) +{ + for (int i = 0; i < tp->num_rx_rings; i++) { + struct rtl8169_rx_ring *ring = &tp->rx_ring[i]; + + memset(ring->rx_desc_array, 0x0, R8169_RX_RING_BYTES); + } +} + static void rtl8169_init_ring_indexes(struct rtl8169_private *tp) { - tp->dirty_tx = tp->cur_tx = tp->cur_rx = 0; + tp->dirty_tx = 0; + tp->cur_tx = 0; + + for (int i = 0; i < tp->num_rx_rings; i++) { + struct rtl8169_rx_ring *ring = &tp->rx_ring[i]; + + ring->cur_rx = 0; + } } static void rtl_jumbo_config(struct rtl8169_private *tp) @@ -2769,6 +2879,29 @@ static void rtl_hw_reset(struct rtl8169_private *tp) rtl_loop_wait_low(tp, &rtl_chipcmd_cond, 100, 100); } +static void rtl8169_init_rss(struct rtl8169_private *tp) +{ + for (int i = 0; i < tp->rss_data->hw_supp_indir_tbl_entries; i++) + tp->rss_data->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, tp->num_rx_rings); + + netdev_rss_key_fill(tp->rss_data->rss_key, RTL_RSS_KEY_SIZE); +} + +static void rtl_setup_rx_params(struct rtl8169_private *tp) +{ + tp->num_rx_rings = 1; + switch (tp->mac_version) { + case RTL_GIGA_MAC_VER_80: + tp->hw_supp_num_rx_queues = R8169_MAX_RX_QUEUES; + tp->rss_data->hw_supp_indir_tbl_entries = RTL_MAX_INDIRECTION_TABLE_ENTRIES; + break; + default: + tp->hw_supp_num_rx_queues = R8169_DEFAULT_RX_QUEUES; + break; + } + tp->init_rx_desc_type = RX_DESC_TYPE_DEFAULT; +} + static void rtl_request_firmware(struct rtl8169_private *tp) { struct rtl_fw *rtl_fw; @@ -2891,8 +3024,62 @@ static void rtl_set_rx_max_size(struct rtl8169_private *tp) RTL_W16(tp, RxMaxSize, R8169_RX_BUF_SIZE + 1); } +static void rtl8169_store_rss_key(struct rtl8169_private *tp) +{ + u8 *rss_key = tp->rss_data->rss_key; + const u16 rss_key_reg = RSS_KEY_REG; + + /* Write RSS key to HW */ + for (int i = 0; i < RTL_RSS_KEY_SIZE; i += sizeof(u32)) + RTL_W32(tp, rss_key_reg + i, get_unaligned_le32(rss_key + i)); +} + +static void rtl8169_store_reta(struct rtl8169_private *tp) +{ + u8 *indir_tbl = tp->rss_data->rss_indir_tbl; + unsigned int i; + + /* Write redirection table to HW */ + for (i = 0; i < tp->rss_data->hw_supp_indir_tbl_entries; i += 4) { + u32 reta = get_unaligned_le32(&indir_tbl[i]); + + RTL_W32(tp, RSS_INDIRECTION_TBL_REG + i, reta); + } +} + +static void rtl8169_set_rss_hash_opt(struct rtl8169_private *tp) +{ + u32 rss_ctrl; + + rss_ctrl = FIELD_PREP(RSS_CPU_NUM_MASK, ilog2(tp->num_rx_rings)); + + /* Perform hash on these packet types */ + rss_ctrl |= RSS_CTRL_TCP_IPV4_SUPP | + RSS_CTRL_IPV4_SUPP | + RSS_CTRL_IPV6_SUPP | + RSS_CTRL_IPV6_EXT_SUPP | + RSS_CTRL_TCP_IPV6_SUPP | + RSS_CTRL_TCP_IPV6_EXT_SUPP | + RSS_CTRL_UDP_IPV4_SUPP | + RSS_CTRL_UDP_IPV6_SUPP; + + rss_ctrl |= FIELD_PREP(RSS_HASH_MASK, + ilog2(tp->rss_data->hw_supp_indir_tbl_entries)); + + RTL_W32(tp, RSS_CTRL_8125, rss_ctrl); +} + +static void rtl_set_rss_config(struct rtl8169_private *tp) +{ + rtl8169_set_rss_hash_opt(tp); + rtl8169_store_reta(tp); + rtl8169_store_rss_key(tp); +} + static void rtl_set_rx_tx_desc_registers(struct rtl8169_private *tp) { + struct rtl8169_rx_ring *ring = &tp->rx_ring[0]; + /* * Magic spell: some iop3xx ARM board needs the TxDescAddrHigh * register to be written before TxDescAddrLow to work. @@ -2900,8 +3087,18 @@ static void rtl_set_rx_tx_desc_registers(struct rtl8169_private *tp) */ RTL_W32(tp, TxDescStartAddrHigh, ((u64) tp->TxPhyAddr) >> 32); RTL_W32(tp, TxDescStartAddrLow, ((u64) tp->TxPhyAddr) & DMA_BIT_MASK(32)); - RTL_W32(tp, RxDescAddrHigh, ((u64) tp->RxPhyAddr) >> 32); - RTL_W32(tp, RxDescAddrLow, ((u64) tp->RxPhyAddr) & DMA_BIT_MASK(32)); + RTL_W32(tp, RxDescAddrHigh, ((u64)ring->rx_phy_addr) >> 32); + RTL_W32(tp, RxDescAddrLow, + ((u64)ring->rx_phy_addr) & DMA_BIT_MASK(32)); + + for (int i = 1; i < tp->num_rx_rings; i++) { + unsigned int rdsar_reg = RDSAR_Q1_LOW + (i - 1) * 8; + struct rtl8169_rx_ring *ring = &tp->rx_ring[i]; + + RTL_W32(tp, rdsar_reg + 4, ((u64)ring->rx_phy_addr >> 32)); + RTL_W32(tp, rdsar_reg, + ((u64)ring->rx_phy_addr) & DMA_BIT_MASK(32)); + } } static void rtl8169_set_magic_reg(struct rtl8169_private *tp) @@ -4012,6 +4209,27 @@ DECLARE_RTL_COND(rtl_mac_ocp_e00e_cond) return r8168_mac_ocp_read(tp, 0xe00e) & BIT(13); } +static void rtl8125_set_rx_q_num(struct rtl8169_private *tp) +{ + u16 rx_q_num; + u16 q_ctrl; + + rx_q_num = ilog2(tp->num_rx_rings); + q_ctrl = RTL_R16(tp, Q_NUM_CTRL_8125); + q_ctrl &= ~RTL_RX_Q_NUM_MASK; + q_ctrl |= FIELD_PREP(RTL_RX_Q_NUM_MASK, rx_q_num); + RTL_W16(tp, Q_NUM_CTRL_8125, q_ctrl); +} + +static void rtl8169_hw_enable_vec_mapping(struct rtl8169_private *tp) +{ + u8 tmp; + + tmp = RTL_R8(tp, INT_CFG0_8125); + tmp |= INT_CFG0_ENABLE_8125; + RTL_W8(tp, INT_CFG0_8125, tmp); +} + static void rtl_hw_start_8125_common(struct rtl8169_private *tp) { rtl_pcie_state_l2l3_disable(tp); @@ -4020,6 +4238,9 @@ static void rtl_hw_start_8125_common(struct rtl8169_private *tp) RTL_W32(tp, RSS_CTRL_8125, 0); RTL_W16(tp, Q_NUM_CTRL_8125, 0); + if (tp->irq_nvecs > 1) + rtl8169_hw_enable_vec_mapping(tp); + /* disable UPS */ r8168_mac_ocp_modify(tp, 0xd40a, 0x0010, 0x0000); @@ -4039,6 +4260,13 @@ static void rtl_hw_start_8125_common(struct rtl8169_private *tp) tp->mac_version == RTL_GIGA_MAC_VER_80) RTL_W8(tp, 0xD8, RTL_R8(tp, 0xD8) & ~0x02); + /* enable rx descriptor type v4 and set queue num for rss */ + if (tp->num_rx_rings > 1) { + rtl8125_set_rx_q_num(tp); + RTL_W8(tp, RTL_DESC_TYPE_CTRL, + RTL_R8(tp, RTL_DESC_TYPE_CTRL) | RTL_DESC_TYPE_RSS); + } + if (tp->mac_version == RTL_GIGA_MAC_VER_80) r8168_mac_ocp_modify(tp, 0xe614, 0x0f00, 0x0f00); else if (tp->mac_version == RTL_GIGA_MAC_VER_70) @@ -4275,6 +4503,12 @@ static void rtl_hw_start(struct rtl8169_private *tp) rtl_hw_aspm_clkreq_enable(tp, true); rtl_set_rx_max_size(tp); rtl_set_rx_tx_desc_registers(tp); + if (rtl_is_8125(tp)) { + if (tp->num_rx_rings > 1) + rtl_set_rss_config(tp); + else + RTL_W32(tp, RSS_CTRL_8125, 0x00); + } rtl_lock_config_regs(tp); rtl_jumbo_config(tp); @@ -4313,19 +4547,32 @@ static int rtl8169_change_mtu(struct net_device *dev, int new_mtu) return 0; } -static void rtl8169_mark_to_asic(struct RxDesc *desc) +static void rtl8169_mark_to_asic(struct rtl8169_private *tp, struct RxDesc *desc) { - u32 eor = le32_to_cpu(desc->opts1) & RingEnd; + u32 eor; - desc->opts2 = 0; - /* Force memory writes to complete before releasing descriptor */ - dma_wmb(); - WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | R8169_RX_BUF_SIZE)); + switch (tp->init_rx_desc_type) { + case RX_DESC_TYPE_RSS: + eor = le32_to_cpu(desc->rss_opts1) & RingEnd; + desc->rss_opts2 = cpu_to_le32(0); + /* Force memory writes to complete before releasing descriptor */ + dma_wmb(); + WRITE_ONCE(desc->rss_opts1, cpu_to_le32(DescOwn | eor | R8169_RX_BUF_SIZE)); + break; + default: + eor = le32_to_cpu(desc->opts1) & RingEnd; + desc->opts2 = cpu_to_le32(0); + /* Force memory writes to complete before releasing descriptor */ + dma_wmb(); + WRITE_ONCE(desc->opts1, cpu_to_le32(DescOwn | eor | R8169_RX_BUF_SIZE)); + break; + } } static struct page *rtl8169_alloc_rx_data(struct rtl8169_private *tp, - struct RxDesc *desc) + struct rtl8169_rx_ring *ring, unsigned int index) { + struct RxDesc *desc = ring->rx_desc_array + index; struct device *d = tp_to_dev(tp); int node = dev_to_node(d); dma_addr_t mapping; @@ -4342,56 +4589,128 @@ static struct page *rtl8169_alloc_rx_data(struct rtl8169_private *tp, return NULL; } - desc->addr = cpu_to_le64(mapping); - rtl8169_mark_to_asic(desc); + ring->rx_buf_dma_addr[index] = mapping; + if (tp->init_rx_desc_type == RX_DESC_TYPE_RSS) + desc->rss_addr = cpu_to_le64(mapping); + else + desc->addr = cpu_to_le64(mapping); + rtl8169_mark_to_asic(tp, desc); return data; } -static void rtl8169_rx_clear(struct rtl8169_private *tp) +static void rtl8169_rx_clear(struct rtl8169_private *tp, + struct rtl8169_rx_ring *ring) { int i; - for (i = 0; i < NUM_RX_DESC && tp->Rx_databuff[i]; i++) { + for (i = 0; i < NUM_RX_DESC && ring->rx_databuff[i]; i++) { dma_unmap_page(tp_to_dev(tp), - le64_to_cpu(tp->RxDescArray[i].addr), + ring->rx_buf_dma_addr[i], R8169_RX_BUF_SIZE, DMA_FROM_DEVICE); - __free_pages(tp->Rx_databuff[i], get_order(R8169_RX_BUF_SIZE)); - tp->Rx_databuff[i] = NULL; - tp->RxDescArray[i].addr = 0; - tp->RxDescArray[i].opts1 = 0; + __free_pages(ring->rx_databuff[i], get_order(R8169_RX_BUF_SIZE)); + ring->rx_databuff[i] = NULL; + ring->rx_buf_dma_addr[i] = 0; + if (tp->init_rx_desc_type == RX_DESC_TYPE_RSS) { + ring->rx_desc_array[i].rss_addr = 0; + ring->rx_desc_array[i].rss_opts1 = 0; + } else { + ring->rx_desc_array[i].addr = 0; + ring->rx_desc_array[i].opts1 = 0; + } } } -static int rtl8169_rx_fill(struct rtl8169_private *tp) +static void rtl8169_mark_as_last_descriptor(struct rtl8169_private *tp, struct RxDesc *desc) +{ + switch (tp->init_rx_desc_type) { + case RX_DESC_TYPE_RSS: + desc->rss_opts1 |= cpu_to_le32(RingEnd); + break; + default: + desc->opts1 |= cpu_to_le32(RingEnd); + break; + } +} + +static int rtl8169_rx_fill(struct rtl8169_private *tp, struct rtl8169_rx_ring *ring) { int i; for (i = 0; i < NUM_RX_DESC; i++) { struct page *data; - data = rtl8169_alloc_rx_data(tp, tp->RxDescArray + i); + data = rtl8169_alloc_rx_data(tp, ring, i); if (!data) { - rtl8169_rx_clear(tp); + rtl8169_rx_clear(tp, ring); return -ENOMEM; } - tp->Rx_databuff[i] = data; + ring->rx_databuff[i] = data; } /* mark as last descriptor in the ring */ - tp->RxDescArray[NUM_RX_DESC - 1].opts1 |= cpu_to_le32(RingEnd); + rtl8169_mark_as_last_descriptor(tp, &ring->rx_desc_array[NUM_RX_DESC - 1]); return 0; } +static int rtl8169_alloc_rx_desc(struct rtl8169_private *tp) +{ + struct pci_dev *pdev = tp->pci_dev; + struct rtl8169_rx_ring *ring; + + for (int i = 0; i < tp->num_rx_rings; i++) { + ring = &tp->rx_ring[i]; + ring->rx_desc_array = dma_alloc_coherent(&pdev->dev, + R8169_RX_RING_BYTES, + &ring->rx_phy_addr, + GFP_KERNEL); + if (!ring->rx_desc_array) + return -ENOMEM; + } + return 0; +} + +static void rtl8169_free_rx_desc(struct rtl8169_private *tp) +{ + struct pci_dev *pdev = tp->pci_dev; + struct rtl8169_rx_ring *ring; + + for (int i = 0; i < tp->num_rx_rings; i++) { + ring = &tp->rx_ring[i]; + if (ring->rx_desc_array) { + dma_free_coherent(&pdev->dev, + R8169_RX_RING_BYTES, + ring->rx_desc_array, + ring->rx_phy_addr); + ring->rx_desc_array = NULL; + } + } +} + static int rtl8169_init_ring(struct rtl8169_private *tp) { + int i, ret; + rtl8169_init_ring_indexes(tp); + rtl8169_rx_desc_init(tp); memset(tp->tx_skb, 0, sizeof(tp->tx_skb)); - memset(tp->Rx_databuff, 0, sizeof(tp->Rx_databuff)); - return rtl8169_rx_fill(tp); + for (i = 0; i < tp->num_rx_rings; i++) { + struct rtl8169_rx_ring *ring = &tp->rx_ring[i]; + + memset(ring->rx_databuff, 0, sizeof(ring->rx_databuff)); + ret = rtl8169_rx_fill(tp, ring); + if (ret < 0) + goto err_clear; + } + return 0; + +err_clear: + while (--i >= 0) + rtl8169_rx_clear(tp, &tp->rx_ring[i]); + return ret; } static void rtl8169_unmap_tx_skb(struct rtl8169_private *tp, unsigned int entry) @@ -4431,9 +4750,31 @@ static void rtl8169_tx_clear(struct rtl8169_private *tp) netdev_reset_queue(tp->dev); } +static void rtl8169_napi_disable(struct rtl8169_private *tp) +{ + if (tp->irq_nvecs > 1) { + for (int i = 0; i < tp->num_rx_rings; i++) + napi_disable(&tp->rtl8169_napi[i]); + napi_disable(&tp->rtl8169_napi[R8169_MAX_RX_QUEUES]); + } else { + napi_disable(&tp->rtl8169_napi[0]); + } +} + +static void rtl8169_napi_enable(struct rtl8169_private *tp) +{ + if (tp->irq_nvecs > 1) { + for (int i = 0; i < tp->num_rx_rings; i++) + napi_enable(&tp->rtl8169_napi[i]); + napi_enable(&tp->rtl8169_napi[R8169_MAX_RX_QUEUES]); + } else { + napi_enable(&tp->rtl8169_napi[0]); + } +} + static void rtl8169_cleanup(struct rtl8169_private *tp) { - napi_disable(&tp->napi); + rtl8169_napi_disable(tp); /* Give a racing hard_start_xmit a few cycles to complete. */ synchronize_net(); @@ -4468,18 +4809,30 @@ static void rtl8169_cleanup(struct rtl8169_private *tp) rtl8169_init_ring_indexes(tp); } +static void rtl8169_rx_desc_reset(struct rtl8169_private *tp) +{ + for (int i = 0; i < tp->num_rx_rings; i++) { + struct rtl8169_rx_ring *ring = &tp->rx_ring[i]; + + for (int j = 0; j < NUM_RX_DESC; j++) { + dma_addr_t phy_addr = ring->rx_buf_dma_addr[j]; + + if (tp->init_rx_desc_type == RX_DESC_TYPE_RSS) + ring->rx_desc_array[j].rss_addr = cpu_to_le64(phy_addr); + rtl8169_mark_to_asic(tp, ring->rx_desc_array + j); + } + } +} + static void rtl_reset_work(struct rtl8169_private *tp) { - int i; - netif_stop_queue(tp->dev); rtl8169_cleanup(tp); - for (i = 0; i < NUM_RX_DESC; i++) - rtl8169_mark_to_asic(tp->RxDescArray + i); + rtl8169_rx_desc_reset(tp); - napi_enable(&tp->napi); + rtl8169_napi_enable(tp); rtl_hw_start(tp); } @@ -4923,30 +5276,115 @@ static inline int rtl8169_fragmented_frame(u32 status) return (status & (FirstFrag | LastFrag)) != (FirstFrag | LastFrag); } -static inline void rtl8169_rx_csum(struct sk_buff *skb, u32 opts1) +static void rtl8169_rx_hash(struct rtl8169_private *tp, + struct RxDesc *desc, + struct sk_buff *skb) { - u32 status = opts1 & (RxProtoMask | RxCSFailMask); + u32 rss_header_info; + u32 hash_val; - if (status == RxProtoTCP || status == RxProtoUDP) + if (!(tp->dev->features & NETIF_F_RXHASH)) + return; + + rss_header_info = le32_to_cpu(desc->rss_dword.rss_info); + + if (!(rss_header_info & RXS_RSS_L3_TYPE_MASK)) + return; + + hash_val = le32_to_cpu(desc->rss_dword.rss_result); + + skb_set_hash(skb, hash_val, + (RXS_RSS_L4_TYPE_MASK & rss_header_info) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); +} + +static void rtl8169_rx_csum(struct rtl8169_private *tp, + struct sk_buff *skb, + u32 opts1) +{ + bool csum_ok = false; + + switch (tp->init_rx_desc_type) { + case RX_DESC_TYPE_RSS: + if (((opts1 & RX_TCPT_DESC_RSS) && !(opts1 & RX_TCPF_DESC_RSS)) || + ((opts1 & RX_UDPT_DESC_RSS) && !(opts1 & RX_UDPF_DESC_RSS))) + csum_ok = true; + break; + default: { + u32 status = opts1 & (RxProtoMask | RxCSFailMask); + + if (status == RxProtoTCP || status == RxProtoUDP) + csum_ok = true; + break; + } + } + + if (csum_ok) skb->ip_summed = CHECKSUM_UNNECESSARY; else skb_checksum_none_assert(skb); } -static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget) +static __le32 rtl8169_rx_desc_opts1(struct rtl8169_private *tp, struct RxDesc *desc) +{ + switch (tp->init_rx_desc_type) { + case RX_DESC_TYPE_RSS: + return READ_ONCE(desc->rss_opts1); + default: + return READ_ONCE(desc->opts1); + } +} + +static bool rtl8169_check_rx_desc_error(struct rtl8169_rx_ring *ring, + struct rtl8169_private *tp, + u32 status) +{ + switch (tp->init_rx_desc_type) { + case RX_DESC_TYPE_RSS: + if (unlikely(status & RX_RES_RSS)) { + u64_stats_update_begin(&ring->stats.syncp); + + /* Note: The RTL8127 RSS descriptor layout does not have + * an RxRWT (Receive Watchdog Timer / over-length) bit. + */ + if (status & RXRUNT_RSS) + ring->stats.rx_length_errors++; + if (status & RXCRC_RSS) + ring->stats.rx_crc_errors++; + u64_stats_update_end(&ring->stats.syncp); + return true; + } + break; + default: + if (unlikely(status & RxRES)) { + u64_stats_update_begin(&ring->stats.syncp); + if (status & (RxRWT | RxRUNT)) + ring->stats.rx_length_errors++; + if (status & RxCRC) + ring->stats.rx_crc_errors++; + u64_stats_update_end(&ring->stats.syncp); + return true; + } + } + return false; +} + +static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, + struct rtl8169_rx_ring *ring, int budget, + struct napi_struct *napi) { struct device *d = tp_to_dev(tp); int count; - for (count = 0; count < budget; count++, tp->cur_rx++) { - unsigned int pkt_size, entry = tp->cur_rx % NUM_RX_DESC; - struct RxDesc *desc = tp->RxDescArray + entry; + for (count = 0; count < budget; count++, ring->cur_rx++) { + unsigned int pkt_size, entry = ring->cur_rx % NUM_RX_DESC; + struct RxDesc *desc = ring->rx_desc_array + entry; struct sk_buff *skb; const void *rx_buf; dma_addr_t addr; u32 status; - status = le32_to_cpu(READ_ONCE(desc->opts1)); + status = le32_to_cpu(rtl8169_rx_desc_opts1(tp, desc)); if (status & DescOwn) break; @@ -4956,23 +5394,31 @@ static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget */ dma_rmb(); - if (unlikely(status & RxRES)) { + if (rtl8169_check_rx_desc_error(ring, tp, status)) { if (net_ratelimit()) netdev_warn(dev, "Rx ERROR. status = %08x\n", status); - dev->stats.rx_errors++; - if (status & (RxRWT | RxRUNT)) - dev->stats.rx_length_errors++; - if (status & RxCRC) - dev->stats.rx_crc_errors++; + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.rx_errors++; + u64_stats_update_end(&ring->stats.syncp); if (!(dev->features & NETIF_F_RXALL)) goto release_descriptor; - else if (status & RxRWT || !(status & (RxRUNT | RxCRC))) - goto release_descriptor; + + if (tp->init_rx_desc_type == RX_DESC_TYPE_DEFAULT) { + if (status & RxRWT || !(status & (RxRUNT | RxCRC))) + goto release_descriptor; + } else { + if (!(status & (RXRUNT_RSS | RXCRC_RSS))) + goto release_descriptor; + } } pkt_size = status & GENMASK(13, 0); + + if (unlikely(pkt_size < ETH_HLEN + ETH_FCS_LEN)) + goto release_descriptor; + if (likely(!(dev->features & NETIF_F_RXFCS))) pkt_size -= ETH_FCS_LEN; @@ -4980,19 +5426,23 @@ static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget * They are seen as a symptom of over-mtu sized frames. */ if (unlikely(rtl8169_fragmented_frame(status))) { - dev->stats.rx_dropped++; - dev->stats.rx_length_errors++; + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.rx_dropped++; + ring->stats.rx_length_errors++; + u64_stats_update_end(&ring->stats.syncp); goto release_descriptor; } - skb = napi_alloc_skb(&tp->napi, pkt_size); + skb = napi_alloc_skb(napi, pkt_size); if (unlikely(!skb)) { - dev->stats.rx_dropped++; + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.rx_dropped++; + u64_stats_update_end(&ring->stats.syncp); goto release_descriptor; } - addr = le64_to_cpu(desc->addr); - rx_buf = page_address(tp->Rx_databuff[entry]); + addr = ring->rx_buf_dma_addr[entry]; + rx_buf = page_address(ring->rx_databuff[entry]); dma_sync_single_for_cpu(d, addr, pkt_size, DMA_FROM_DEVICE); prefetch(rx_buf); @@ -5001,19 +5451,28 @@ static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget skb->len = pkt_size; dma_sync_single_for_device(d, addr, pkt_size, DMA_FROM_DEVICE); - rtl8169_rx_csum(skb, status); + if (tp->num_rx_rings > 1) { + rtl8169_rx_hash(tp, desc, skb); + skb_record_rx_queue(skb, ring - tp->rx_ring); + } + rtl8169_rx_csum(tp, skb, status); skb->protocol = eth_type_trans(skb, dev); - rtl8169_rx_vlan_tag(desc, skb); + rtl8169_rx_vlan_tag(tp, desc, skb); - if (skb->pkt_type == PACKET_MULTICAST) - dev->stats.multicast++; + if (skb->pkt_type == PACKET_MULTICAST) { + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.multicast++; + u64_stats_update_end(&ring->stats.syncp); + } - napi_gro_receive(&tp->napi, skb); + napi_gro_receive(napi, skb); dev_sw_netstats_rx_add(dev, pkt_size); release_descriptor: - rtl8169_mark_to_asic(desc); + if (tp->init_rx_desc_type == RX_DESC_TYPE_RSS) + desc->rss_addr = cpu_to_le64(ring->rx_buf_dma_addr[entry]); + rtl8169_mark_to_asic(tp, desc); } return count; @@ -5021,8 +5480,12 @@ static int rtl_rx(struct net_device *dev, struct rtl8169_private *tp, int budget static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance) { - struct rtl8169_private *tp = dev_instance; - u32 status = rtl_get_events(tp); + struct napi_struct *napi = dev_instance; + struct rtl8169_private *tp; + u32 status; + + tp = netdev_priv(napi->dev); + status = rtl_get_events(tp); if ((status & 0xffff) == 0xffff || !(status & tp->irq_mask)) return IRQ_NONE; @@ -5043,13 +5506,127 @@ static irqreturn_t rtl8169_interrupt(int irq, void *dev_instance) } rtl_irq_disable(tp); - napi_schedule(&tp->napi); + napi_schedule(napi); out: rtl_ack_events(tp, status); return IRQ_HANDLED; } +static void rtl8169_free_one_irq(struct rtl8169_private *tp, int i) +{ + struct pci_dev *pdev = tp->pci_dev; + struct napi_struct *napi; + int irq; + + irq = pci_irq_vector(pdev, i); + napi = &tp->rtl8169_napi[i]; + + if (tp->irq_nvecs > 1) { + if (i < tp->num_rx_rings || i == R8169_MAX_RX_QUEUES) + free_irq(irq, napi); + else if (i == MSIX_ID_VEC_MAP_LINKCHG) + free_irq(irq, tp); + } else { + free_irq(irq, napi); + } +} + +static void rtl8169_free_irq(struct rtl8169_private *tp) +{ + int i; + + for (i = 0; i < tp->irq_nvecs; i++) + rtl8169_free_one_irq(tp, i); +} + +static void rtl8169_disable_hw_interrupt_msix(struct rtl8169_private *tp, + int message_id) +{ + RTL_W32(tp, IMR_CLEAR_VEC_MAP_REG, BIT(message_id)); +} + +static void rtl8169_clear_hw_isr(struct rtl8169_private *tp, int message_id) +{ + RTL_W32(tp, ISR_VEC_MAP_REG, BIT(message_id)); +} + +static void rtl8169_enable_hw_interrupt_msix(struct rtl8169_private *tp, + int message_id) +{ + RTL_W32(tp, IMR_SET_VEC_MAP_REG, BIT(message_id)); +} + +static irqreturn_t rtl8169_interrupt_msix(int irq, void *dev_instance) +{ + struct napi_struct *napi = dev_instance; + struct net_device *dev = napi->dev; + struct rtl8169_private *tp; + int message_id; + + tp = netdev_priv(dev); + message_id = napi - tp->rtl8169_napi; + + rtl8169_disable_hw_interrupt_msix(tp, message_id); + rtl8169_clear_hw_isr(tp, message_id); + + napi_schedule(napi); + + return IRQ_HANDLED; +} + +static irqreturn_t rtl8169_interrupt_other(int irq, void *dev_instance) +{ + struct rtl8169_private *tp = dev_instance; + + rtl8169_clear_hw_isr(tp, MSIX_ID_VEC_MAP_LINKCHG); + if (tp->phydev) + phy_mac_interrupt(tp->phydev); + else if (tp->sfp_mode) + phylink_mac_change(tp->phylink, + !!(RTL_R8(tp, PHYstatus) & LinkStatus)); + return IRQ_HANDLED; +} + +static int rtl8169_request_irq(struct rtl8169_private *tp) +{ + struct pci_dev *pdev = tp->pci_dev; + struct net_device *dev = tp->dev; + struct napi_struct *napi; + unsigned long irqflags; + int i, rc, irq; + + irqflags = pci_dev_msi_enabled(pdev) ? IRQF_NO_THREAD : IRQF_SHARED; + + for (i = 0; i < tp->irq_nvecs; i++) { + napi = &tp->rtl8169_napi[i]; + irq = pci_irq_vector(pdev, i); + + if (tp->irq_nvecs > 1) { + /* RX: 0~7, TX: 8*/ + if (i < tp->num_rx_rings || i == R8169_MAX_RX_QUEUES) + rc = request_irq(irq, rtl8169_interrupt_msix, + irqflags, dev->name, napi); + else if (i == MSIX_ID_VEC_MAP_LINKCHG) + rc = request_irq(irq, rtl8169_interrupt_other, + irqflags, dev->name, tp); + else + continue; + } else { + rc = request_irq(irq, rtl8169_interrupt, + irqflags, dev->name, napi); + } + if (rc) + goto free_irq; + } + return 0; + +free_irq: + while (--i >= 0) + rtl8169_free_one_irq(tp, i); + return rc; +} + static void rtl_task(struct work_struct *work) { struct rtl8169_private *tp = @@ -5084,13 +5661,14 @@ static void rtl_task(struct work_struct *work) static int rtl8169_poll(struct napi_struct *napi, int budget) { - struct rtl8169_private *tp = container_of(napi, struct rtl8169_private, napi); - struct net_device *dev = tp->dev; - int work_done; + struct rtl8169_private *tp = netdev_priv(napi->dev); + struct net_device *dev = napi->dev; + int work_done = 0; rtl_tx(dev, tp, budget); - work_done = rtl_rx(dev, tp, budget); + /* rtl8169_poll() is used only when there is a single RX ring. */ + work_done = rtl_rx(dev, tp, &tp->rx_ring[0], budget, napi); if (work_done < budget && napi_complete_done(napi, work_done)) rtl_irq_enable(tp); @@ -5175,7 +5753,7 @@ static void rtl8169_up(struct rtl8169_private *tp) if (tp->phydev) rtl8169_init_phy(tp); - napi_enable(&tp->napi); + rtl8169_napi_enable(tp); enable_work(&tp->wk.work); rtl_reset_work(tp); } @@ -5190,18 +5768,17 @@ static int rtl8169_close(struct net_device *dev) phylink_stop(tp->phylink); netif_stop_queue(dev); rtl8169_down(tp); - rtl8169_rx_clear(tp); + for (int i = 0; i < tp->num_rx_rings; i++) + rtl8169_rx_clear(tp, &tp->rx_ring[i]); - free_irq(tp->irq, tp); + rtl8169_free_irq(tp); phylink_disconnect_phy(tp->phylink); - dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray, - tp->RxPhyAddr); dma_free_coherent(&pdev->dev, R8169_TX_RING_BYTES, tp->TxDescArray, tp->TxPhyAddr); tp->TxDescArray = NULL; - tp->RxDescArray = NULL; + rtl8169_free_rx_desc(tp); pm_runtime_put_sync(&pdev->dev); @@ -5213,7 +5790,13 @@ static void rtl8169_netpoll(struct net_device *dev) { struct rtl8169_private *tp = netdev_priv(dev); - rtl8169_interrupt(tp->irq, tp); + if (tp->irq_nvecs > 1) { + for (int i = 0; i < tp->num_rx_rings; i++) + rtl8169_interrupt_msix(0, &tp->rtl8169_napi[i]); + rtl8169_interrupt_msix(0, &tp->rtl8169_napi[R8169_MAX_RX_QUEUES]); + } else { + rtl8169_interrupt(0, &tp->rtl8169_napi[0]); + } } #endif @@ -5221,7 +5804,6 @@ static int rtl_open(struct net_device *dev) { struct rtl8169_private *tp = netdev_priv(dev); struct pci_dev *pdev = tp->pci_dev; - unsigned long irqflags; int retval = -ENOMEM; pm_runtime_get_sync(&pdev->dev); @@ -5235,10 +5817,8 @@ static int rtl_open(struct net_device *dev) if (!tp->TxDescArray) goto out; - tp->RxDescArray = dma_alloc_coherent(&pdev->dev, R8169_RX_RING_BYTES, - &tp->RxPhyAddr, GFP_KERNEL); - if (!tp->RxDescArray) - goto err_free_tx_0; + if (rtl8169_alloc_rx_desc(tp) < 0) + goto err_free_rx_1; retval = rtl8169_init_ring(tp); if (retval < 0) @@ -5246,8 +5826,7 @@ static int rtl_open(struct net_device *dev) rtl_request_firmware(tp); - irqflags = pci_dev_msi_enabled(pdev) ? IRQF_NO_THREAD : IRQF_SHARED; - retval = request_irq(tp->irq, rtl8169_interrupt, irqflags, dev->name, tp); + retval = rtl8169_request_irq(tp); if (retval < 0) goto err_release_fw_2; @@ -5266,15 +5845,13 @@ static int rtl_open(struct net_device *dev) return retval; err_free_irq: - free_irq(tp->irq, tp); + rtl8169_free_irq(tp); err_release_fw_2: rtl_release_firmware(tp); - rtl8169_rx_clear(tp); + for (int i = 0; i < tp->num_rx_rings; i++) + rtl8169_rx_clear(tp, &tp->rx_ring[i]); err_free_rx_1: - dma_free_coherent(&pdev->dev, R8169_RX_RING_BYTES, tp->RxDescArray, - tp->RxPhyAddr); - tp->RxDescArray = NULL; -err_free_tx_0: + rtl8169_free_rx_desc(tp); dma_free_coherent(&pdev->dev, R8169_TX_RING_BYTES, tp->TxDescArray, tp->TxPhyAddr); tp->TxDescArray = NULL; @@ -5293,6 +5870,27 @@ rtl8169_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) netdev_stats_to_stats64(stats, &dev->stats); dev_fetch_sw_netstats(stats, dev->tstats); + for (int i = 0; i < tp->num_rx_rings; i++) { + u64 errors, dropped, length_errors, crc_errors, multicast; + struct rtl8169_rx_ring *ring = &tp->rx_ring[i]; + unsigned int start; + + do { + start = u64_stats_fetch_begin(&ring->stats.syncp); + errors = ring->stats.rx_errors; + dropped = ring->stats.rx_dropped; + length_errors = ring->stats.rx_length_errors; + crc_errors = ring->stats.rx_crc_errors; + multicast = ring->stats.multicast; + } while (u64_stats_fetch_retry(&ring->stats.syncp, start)); + + stats->rx_errors += errors; + stats->rx_dropped += dropped; + stats->rx_length_errors += length_errors; + stats->rx_crc_errors += crc_errors; + stats->multicast += multicast; + } + /* * Fetch additional counter values missing in stats collected by driver * from tally counters. @@ -5419,6 +6017,19 @@ static void rtl_shutdown(struct pci_dev *pdev) pci_prepare_to_sleep(pdev); } +static void r8169_free_napi(struct rtl8169_private *tp) +{ + if (tp->irq_nvecs > 1) { + for (int i = 0; i < tp->num_rx_rings; i++) + netif_napi_del(&tp->rtl8169_napi[i]); + netif_napi_del(&tp->rtl8169_napi[R8169_MAX_RX_QUEUES]); + } else { + netif_napi_del(&tp->rtl8169_napi[0]); + } + + kfree(tp->rtl8169_napi); +} + static void rtl_remove_one(struct pci_dev *pdev) { struct rtl8169_private *tp = pci_get_drvdata(pdev); @@ -5432,6 +6043,8 @@ static void rtl_remove_one(struct pci_dev *pdev) r8169_remove_leds(tp->leds); unregister_netdev(tp->dev); + r8169_free_napi(tp); + kfree(tp->rx_ring); phylink_destroy(tp->phylink); if (tp->dash_type != RTL_DASH_NONE) @@ -5465,15 +6078,51 @@ static const struct net_device_ops rtl_netdev_ops = { static void rtl_set_irq_mask(struct rtl8169_private *tp) { - tp->irq_mask = RxOK | RxErr | TxOK | TxErr | LinkChg; + if (tp->irq_nvecs > 1) { + tp->irq_mask = ISRIMR_LINKCHG | ISRIMR_TOK_Q0; + for (int i = 0; i < tp->num_rx_rings; i++) + tp->irq_mask |= ISRIMR_ROK_Q0 << i; + } else { + tp->irq_mask = RxOK | RxErr | TxOK | TxErr | LinkChg; - if (tp->mac_version <= RTL_GIGA_MAC_VER_06) - tp->irq_mask |= SYSErr | RxFIFOOver; + if (tp->mac_version <= RTL_GIGA_MAC_VER_06) + tp->irq_mask |= SYSErr | RxFIFOOver; + } +} + +static int get_max_irq_nvecs(struct rtl8169_private *tp) +{ + if (tp->mac_version == RTL_GIGA_MAC_VER_80) + return R8127_MAX_NUM_IRQVEC; + return R8169_IRQ_DEFAULT; +} + +static int get_min_irq_nvecs(struct rtl8169_private *tp) +{ + if (tp->mac_version == RTL_GIGA_MAC_VER_80) + return R8127_MIN_NUM_IRQVEC; + return R8169_IRQ_DEFAULT; +} + +static void rtl8169_set_rx_ring_num(struct rtl8169_private *tp) +{ + if (tp->irq_nvecs >= get_min_irq_nvecs(tp)) { + unsigned int rss_queue_num = netif_get_num_default_rss_queues(); + + tp->num_rx_rings = rounddown_pow_of_two(min(rss_queue_num, + tp->hw_supp_num_rx_queues)); + if (tp->num_rx_rings >= 2) + tp->init_rx_desc_type = RX_DESC_TYPE_RSS; + } } static int rtl_alloc_irq(struct rtl8169_private *tp) { + struct pci_dev *pdev = tp->pci_dev; unsigned int flags; + int nvecs, min_nvecs; + + min_nvecs = get_min_irq_nvecs(tp); switch (tp->mac_version) { case RTL_GIGA_MAC_VER_02 ... RTL_GIGA_MAC_VER_06: @@ -5489,7 +6138,21 @@ static int rtl_alloc_irq(struct rtl8169_private *tp) break; } - return pci_alloc_irq_vectors(tp->pci_dev, 1, 1, flags); + nvecs = pci_alloc_irq_vectors(pdev, min_nvecs, + get_max_irq_nvecs(tp), flags); + + if (nvecs < 0 && min_nvecs > 1) { + dev_info(&pdev->dev, "Failed to allocate %d MSI-X vectors.\n", + min_nvecs); + nvecs = pci_alloc_irq_vectors(pdev, 1, 1, flags); + } + + if (nvecs < 0) + return nvecs; + + tp->irq_nvecs = nvecs; + + return 0; } static void rtl_read_mac_address(struct rtl8169_private *tp, @@ -6012,6 +6675,119 @@ static int rtl_init_phylink(struct rtl8169_private *tp) return 0; } +static int rtl8169_poll_msix_rx(struct napi_struct *napi, int budget) +{ + struct net_device *dev = napi->dev; + struct rtl8169_private *tp; + int work_done = 0; + int message_id; + + tp = netdev_priv(dev); + message_id = napi - tp->rtl8169_napi; + + if (message_id < tp->num_rx_rings) + work_done += rtl_rx(dev, tp, &tp->rx_ring[message_id], + budget, napi); + + if (work_done < budget && napi_complete_done(napi, work_done)) + rtl8169_enable_hw_interrupt_msix(tp, message_id); + + return work_done; +} + +static int rtl8169_poll_msix_tx(struct napi_struct *napi, int budget) +{ + struct net_device *dev = napi->dev; + struct rtl8169_private *tp; + + tp = netdev_priv(dev); + + /* Currently r8169 only supports a single Tx ring. + * Therefore, we don't need a per-ring Tx processing loop here. + */ + rtl_tx(dev, tp, budget); + + if (budget && napi_complete_done(napi, 0)) + rtl8169_enable_hw_interrupt_msix(tp, (int)(napi - tp->rtl8169_napi)); + + return 0; +} + +/* RTL8127 MSI-X vector layout: + * Vectors 0 .. (MAX_RXQ - 1) : Rx Queues + * Vectors MAX_RXQ .. (MAX_RXQ + MAX_TXQ - 1) : Tx Queues + * NAPI is only allocated for data path + */ +static void r8169_init_napi(struct rtl8169_private *tp) +{ + int i; + + if (tp->irq_nvecs > 1) { + for (i = 0; i < tp->num_rx_rings; i++) { + netif_napi_add(tp->dev, &tp->rtl8169_napi[i], + rtl8169_poll_msix_rx); + netif_napi_set_irq(&tp->rtl8169_napi[i], + pci_irq_vector(tp->pci_dev, i)); + } + netif_napi_add_tx(tp->dev, + &tp->rtl8169_napi[R8169_MAX_RX_QUEUES], + rtl8169_poll_msix_tx); + netif_napi_set_irq(&tp->rtl8169_napi[R8169_MAX_RX_QUEUES], + pci_irq_vector(tp->pci_dev, R8169_MAX_RX_QUEUES)); + } else { + netif_napi_add(tp->dev, &tp->rtl8169_napi[0], rtl8169_poll); + netif_napi_set_irq(&tp->rtl8169_napi[0], + pci_irq_vector(tp->pci_dev, 0)); + } +} + +static void rtl8169_get_channels(struct net_device *dev, + struct ethtool_channels *ch) +{ + struct rtl8169_private *tp = netdev_priv(dev); + + if (tp->irq_nvecs > 1) { + ch->max_rx = tp->hw_supp_num_rx_queues; + ch->max_tx = 1; + ch->max_other = 1; + ch->rx_count = tp->num_rx_rings; + ch->tx_count = 1; + ch->other_count = 1; + } else { + ch->max_combined = 1; + ch->combined_count = 1; + } +} + +static const struct ethtool_ops rtl8169_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_MAX_FRAMES, + .get_drvinfo = rtl8169_get_drvinfo, + .get_regs_len = rtl8169_get_regs_len, + .get_link = ethtool_op_get_link, + .get_coalesce = rtl_get_coalesce, + .set_coalesce = rtl_set_coalesce, + .get_regs = rtl8169_get_regs, + .get_wol = rtl8169_get_wol, + .set_wol = rtl8169_set_wol, + .get_strings = rtl8169_get_strings, + .get_sset_count = rtl8169_get_sset_count, + .get_ethtool_stats = rtl8169_get_ethtool_stats, + .get_ts_info = ethtool_op_get_ts_info, + .nway_reset = rtl8169_nway_reset, + .get_eee = rtl8169_get_eee, + .set_eee = rtl8169_set_eee, + .get_channels = rtl8169_get_channels, + .get_link_ksettings = rtl8169_get_link_ksettings, + .set_link_ksettings = rtl8169_set_link_ksettings, + .get_ringparam = rtl8169_get_ringparam, + .get_pause_stats = rtl8169_get_pause_stats, + .get_pauseparam = rtl8169_get_pauseparam, + .set_pauseparam = rtl8169_set_pauseparam, + .get_eth_mac_stats = rtl8169_get_eth_mac_stats, + .get_eth_ctrl_stats = rtl8169_get_eth_ctrl_stats, +}; + static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { const struct rtl_chip_info *chip; @@ -6022,7 +6798,10 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) u32 txconfig; u32 xid; - dev = devm_alloc_etherdev(&pdev->dev, sizeof (*tp)); + dev = devm_alloc_etherdev_mqs(&pdev->dev, sizeof(*tp), + R8169_MAX_TX_QUEUES, + R8169_MAX_RX_QUEUES); + if (!dev) return -ENOMEM; @@ -6098,6 +6877,13 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) tp->dash_type = rtl_get_dash_type(tp); tp->dash_enabled = rtl_dash_is_enabled(tp); + if (rtl_hw_support_rss(tp)) { + tp->rss_data = devm_kzalloc(&pdev->dev, sizeof(*tp->rss_data), + GFP_KERNEL); + if (!tp->rss_data) + return -ENOMEM; + } + tp->cp_cmd = RTL_R16(tp, CPlusCmd) & CPCMD_MASK; if (sizeof(dma_addr_t) > 4 && tp->mac_version >= RTL_GIGA_MAC_VER_18 && @@ -6112,11 +6898,16 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) rtl_hw_reset(tp); + rtl_setup_rx_params(tp); + rc = rtl_alloc_irq(tp); if (rc < 0) return dev_err_probe(&pdev->dev, rc, "Can't allocate interrupt\n"); - tp->irq = pci_irq_vector(pdev, 0); + rtl8169_set_rx_ring_num(tp); + + if (rtl_hw_support_rss(tp)) + rtl8169_init_rss(tp); INIT_WORK(&tp->wk.work, rtl_task); disable_work(&tp->wk.work); @@ -6125,13 +6916,16 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->ethtool_ops = &rtl8169_ethtool_ops; - netif_napi_add(dev, &tp->napi, rtl8169_poll); - dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; + if (rtl_hw_support_rss(tp) && tp->num_rx_rings > 1) { + dev->hw_features |= NETIF_F_RXHASH; + dev->features |= NETIF_F_RXHASH; + } + /* * Pretend we are using VLANs; This bypasses a nasty bug where * Interrupts stop flowing on high load on 8110SCd controllers. @@ -6187,6 +6981,10 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (jumbo_max) dev->max_mtu = jumbo_max; + rc = netif_set_real_num_queues(tp->dev, 1, tp->num_rx_rings); + if (rc < 0) + return dev_err_probe(&pdev->dev, rc, "set tx/rx num failure\n"); + rtl_set_irq_mask(tp); tp->counters = dmam_alloc_coherent (&pdev->dev, sizeof(*tp->counters), @@ -6209,12 +7007,29 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } } - rc = register_netdev(dev); - if (rc) { - phylink_destroy(tp->phylink); - return rc; + tp->rx_ring = kcalloc(tp->num_rx_rings, sizeof(struct rtl8169_rx_ring), + GFP_KERNEL); + if (!tp->rx_ring) { + rc = -ENOMEM; + goto err_destroy_phylink; } + for (int i = 0; i < tp->num_rx_rings; i++) + u64_stats_init(&tp->rx_ring[i].stats.syncp); + + tp->rtl8169_napi = kcalloc(tp->irq_nvecs, sizeof(struct napi_struct), + GFP_KERNEL); + if (!tp->rtl8169_napi) { + rc = -ENOMEM; + goto err_free_rx_ring; + } + + r8169_init_napi(tp); + + rc = register_netdev(dev); + if (rc) + goto err_free_napi; + if (IS_ENABLED(CONFIG_R8169_LEDS)) { if (rtl_is_8125(tp)) tp->leds = rtl8125_init_leds(dev); @@ -6222,8 +7037,9 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) tp->leds = rtl8168_init_leds(dev); } - netdev_info(dev, "%s, %pM, %sXID %x, IRQ %d\n", - chip->name, dev->dev_addr, ext_xid_str, xid, tp->irq); + netdev_info(dev, "%s, %pM, %sXID %x, IRQ %d (%u total)\n", + chip->name, dev->dev_addr, ext_xid_str, xid, + pci_irq_vector(pdev, 0), tp->irq_nvecs); if (jumbo_max) netdev_info(dev, "jumbo features [frames: %d bytes, tx checksumming: %s]\n", @@ -6243,6 +7059,14 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) pm_runtime_put_sync(&pdev->dev); return 0; + +err_free_napi: + r8169_free_napi(tp); +err_free_rx_ring: + kfree(tp->rx_ring); +err_destroy_phylink: + phylink_destroy(tp->phylink); + return rc; } static struct pci_driver rtl8169_pci_driver = {