mirror of
https://github.com/torvalds/linux.git
synced 2026-09-23 13:14:02 +02:00
Merge branch 'net-mana-add-ethtool-private-flag-for-full-page-rx-buffers'
Dipayaan Roy says: ==================== net: mana: add ethtool private flag for full-page RX buffers On some ARM64 platforms with 4K PAGE_SIZE, utilizing page_pool fragments for allocation in the RX refill path (~2kB buffer per fragment) causes 15-20% throughput regression under high connection counts (>16 TCP streams at 180+ Gbps). Using full-page buffers on these platforms shows no regression and restores line-rate performance. This behavior is observed on a single platform; other platforms perform better with page_pool fragments, indicating this is not a page_pool issue but platform-specific. This series adds an ethtool private flag "full-page-rx" to let the user opt in to one RX buffer per page: ethtool --set-priv-flags eth0 full-page-rx on There is no behavioral change by default. The flag can be persisted via udev rule for affected platforms. ==================== Link: https://patch.msgid.link/20260729063347.3388035-1-dipayanroy@linux.microsoft.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski
commit
5b4f243f78
|
|
@ -755,6 +755,25 @@ static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da)
|
|||
return va;
|
||||
}
|
||||
|
||||
static bool
|
||||
mana_use_single_rxbuf_per_page(struct mana_port_context *apc, u32 mtu)
|
||||
{
|
||||
/* On some platforms with 4K PAGE_SIZE, page_pool fragment allocation
|
||||
* in the RX refill path (~2kB buffer) can cause significant throughput
|
||||
* regression under high connection counts. Allow user to force one RX
|
||||
* buffer per page via ethtool private flag to bypass the fragment
|
||||
* path.
|
||||
*/
|
||||
if (apc->priv_flags & BIT(MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF))
|
||||
return true;
|
||||
|
||||
/* For xdp and jumbo frames make sure only one packet fits per page. */
|
||||
if (mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || mana_xdp_get(apc))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Get RX buffer's data size, alloc size, XDP headroom based on MTU */
|
||||
static void mana_get_rxbuf_cfg(struct mana_port_context *apc,
|
||||
int mtu, u32 *datasize, u32 *alloc_size,
|
||||
|
|
@ -765,8 +784,7 @@ static void mana_get_rxbuf_cfg(struct mana_port_context *apc,
|
|||
/* Calculate datasize first (consistent across all cases) */
|
||||
*datasize = mtu + ETH_HLEN;
|
||||
|
||||
/* For xdp and jumbo frames make sure only one packet fits per page */
|
||||
if (mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || mana_xdp_get(apc)) {
|
||||
if (mana_use_single_rxbuf_per_page(apc, mtu)) {
|
||||
if (mana_xdp_get(apc)) {
|
||||
*headroom = XDP_PACKET_HEADROOM;
|
||||
*alloc_size = PAGE_SIZE;
|
||||
|
|
|
|||
|
|
@ -133,58 +133,94 @@ static const struct mana_stats_desc mana_phy_stats[] = {
|
|||
{ "hc_tc7_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc7_phy) },
|
||||
};
|
||||
|
||||
static const char mana_priv_flags[MANA_PRIV_FLAG_MAX][ETH_GSTRING_LEN] = {
|
||||
[MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF] = "full-page-rx"
|
||||
};
|
||||
|
||||
static int mana_get_sset_count(struct net_device *ndev, int stringset)
|
||||
{
|
||||
struct mana_port_context *apc = netdev_priv(ndev);
|
||||
unsigned int num_queues = apc->num_queues;
|
||||
|
||||
if (stringset != ETH_SS_STATS)
|
||||
return -EINVAL;
|
||||
switch (stringset) {
|
||||
case ETH_SS_STATS:
|
||||
return ARRAY_SIZE(mana_eth_stats) +
|
||||
ARRAY_SIZE(mana_phy_stats) +
|
||||
ARRAY_SIZE(mana_hc_stats) +
|
||||
num_queues * (MANA_STATS_RX_COUNT + MANA_STATS_TX_COUNT);
|
||||
|
||||
return ARRAY_SIZE(mana_eth_stats) + ARRAY_SIZE(mana_phy_stats) + ARRAY_SIZE(mana_hc_stats) +
|
||||
num_queues * (MANA_STATS_RX_COUNT + MANA_STATS_TX_COUNT);
|
||||
case ETH_SS_PRIV_FLAGS:
|
||||
return MANA_PRIV_FLAG_MAX;
|
||||
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
static void mana_get_strings_stats(struct mana_port_context *apc, u8 **data)
|
||||
{
|
||||
unsigned int num_queues = apc->num_queues;
|
||||
int i, j;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(mana_eth_stats); i++)
|
||||
ethtool_puts(data, mana_eth_stats[i].name);
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(mana_hc_stats); i++)
|
||||
ethtool_puts(data, mana_hc_stats[i].name);
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(mana_phy_stats); i++)
|
||||
ethtool_puts(data, mana_phy_stats[i].name);
|
||||
|
||||
for (i = 0; i < num_queues; i++) {
|
||||
ethtool_sprintf(data, "rx_%d_packets", i);
|
||||
ethtool_sprintf(data, "rx_%d_bytes", i);
|
||||
ethtool_sprintf(data, "rx_%d_xdp_drop", i);
|
||||
ethtool_sprintf(data, "rx_%d_xdp_tx", i);
|
||||
ethtool_sprintf(data, "rx_%d_xdp_redirect", i);
|
||||
ethtool_sprintf(data, "rx_%d_pkt_len0_err", i);
|
||||
for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
|
||||
ethtool_sprintf(data,
|
||||
"rx_%d_coalesced_cqe_%d",
|
||||
i,
|
||||
j + 2);
|
||||
}
|
||||
|
||||
for (i = 0; i < num_queues; i++) {
|
||||
ethtool_sprintf(data, "tx_%d_packets", i);
|
||||
ethtool_sprintf(data, "tx_%d_bytes", i);
|
||||
ethtool_sprintf(data, "tx_%d_xdp_xmit", i);
|
||||
ethtool_sprintf(data, "tx_%d_tso_packets", i);
|
||||
ethtool_sprintf(data, "tx_%d_tso_bytes", i);
|
||||
ethtool_sprintf(data, "tx_%d_tso_inner_packets", i);
|
||||
ethtool_sprintf(data, "tx_%d_tso_inner_bytes", i);
|
||||
ethtool_sprintf(data, "tx_%d_long_pkt_fmt", i);
|
||||
ethtool_sprintf(data, "tx_%d_short_pkt_fmt", i);
|
||||
ethtool_sprintf(data, "tx_%d_csum_partial", i);
|
||||
ethtool_sprintf(data, "tx_%d_mana_map_err", i);
|
||||
}
|
||||
}
|
||||
|
||||
static void mana_get_strings_priv_flags(u8 **data)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < MANA_PRIV_FLAG_MAX; i++)
|
||||
ethtool_puts(data, mana_priv_flags[i]);
|
||||
}
|
||||
|
||||
static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
|
||||
{
|
||||
struct mana_port_context *apc = netdev_priv(ndev);
|
||||
unsigned int num_queues = apc->num_queues;
|
||||
int i, j;
|
||||
|
||||
if (stringset != ETH_SS_STATS)
|
||||
return;
|
||||
for (i = 0; i < ARRAY_SIZE(mana_eth_stats); i++)
|
||||
ethtool_puts(&data, mana_eth_stats[i].name);
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(mana_hc_stats); i++)
|
||||
ethtool_puts(&data, mana_hc_stats[i].name);
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(mana_phy_stats); i++)
|
||||
ethtool_puts(&data, mana_phy_stats[i].name);
|
||||
|
||||
for (i = 0; i < num_queues; i++) {
|
||||
ethtool_sprintf(&data, "rx_%d_packets", i);
|
||||
ethtool_sprintf(&data, "rx_%d_bytes", i);
|
||||
ethtool_sprintf(&data, "rx_%d_xdp_drop", i);
|
||||
ethtool_sprintf(&data, "rx_%d_xdp_tx", i);
|
||||
ethtool_sprintf(&data, "rx_%d_xdp_redirect", i);
|
||||
ethtool_sprintf(&data, "rx_%d_pkt_len0_err", i);
|
||||
for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
|
||||
ethtool_sprintf(&data, "rx_%d_coalesced_cqe_%d", i, j + 2);
|
||||
}
|
||||
|
||||
for (i = 0; i < num_queues; i++) {
|
||||
ethtool_sprintf(&data, "tx_%d_packets", i);
|
||||
ethtool_sprintf(&data, "tx_%d_bytes", i);
|
||||
ethtool_sprintf(&data, "tx_%d_xdp_xmit", i);
|
||||
ethtool_sprintf(&data, "tx_%d_tso_packets", i);
|
||||
ethtool_sprintf(&data, "tx_%d_tso_bytes", i);
|
||||
ethtool_sprintf(&data, "tx_%d_tso_inner_packets", i);
|
||||
ethtool_sprintf(&data, "tx_%d_tso_inner_bytes", i);
|
||||
ethtool_sprintf(&data, "tx_%d_long_pkt_fmt", i);
|
||||
ethtool_sprintf(&data, "tx_%d_short_pkt_fmt", i);
|
||||
ethtool_sprintf(&data, "tx_%d_csum_partial", i);
|
||||
ethtool_sprintf(&data, "tx_%d_mana_map_err", i);
|
||||
switch (stringset) {
|
||||
case ETH_SS_STATS:
|
||||
mana_get_strings_stats(apc, &data);
|
||||
break;
|
||||
case ETH_SS_PRIV_FLAGS:
|
||||
mana_get_strings_priv_flags(&data);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -739,6 +775,78 @@ static int mana_get_link_ksettings(struct net_device *ndev,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static u32 mana_get_priv_flags(struct net_device *ndev)
|
||||
{
|
||||
struct mana_port_context *apc = netdev_priv(ndev);
|
||||
|
||||
return apc->priv_flags;
|
||||
}
|
||||
|
||||
static int mana_set_priv_flags(struct net_device *ndev, u32 priv_flags)
|
||||
{
|
||||
struct mana_port_context *apc = netdev_priv(ndev);
|
||||
u32 changed = apc->priv_flags ^ priv_flags;
|
||||
u32 old_priv_flags = apc->priv_flags;
|
||||
int err = 0;
|
||||
|
||||
if (!changed)
|
||||
return 0;
|
||||
|
||||
/* Reject unknown bits */
|
||||
if (priv_flags & ~GENMASK(MANA_PRIV_FLAG_MAX - 1, 0))
|
||||
return -EINVAL;
|
||||
|
||||
apc->priv_flags = priv_flags;
|
||||
|
||||
if (changed & BIT(MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF)) {
|
||||
if (!apc->port_is_up)
|
||||
return 0;
|
||||
|
||||
/* If XDP is attached or MTU is jumbo, single-buffer-per-page
|
||||
* is already forced regardless of this flag. Skip the
|
||||
* expensive detach/attach cycle since nothing changes.
|
||||
*/
|
||||
if (ndev->mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 ||
|
||||
mana_xdp_get(apc))
|
||||
return 0;
|
||||
|
||||
/* Block RDMA from grabbing the vport during detach/attach */
|
||||
mutex_lock(&apc->vport_mutex);
|
||||
apc->channel_changing = true;
|
||||
mutex_unlock(&apc->vport_mutex);
|
||||
|
||||
err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues);
|
||||
if (err) {
|
||||
netdev_err(ndev,
|
||||
"Insufficient memory for new allocations\n");
|
||||
apc->priv_flags = old_priv_flags;
|
||||
goto clear_flag;
|
||||
}
|
||||
|
||||
err = mana_detach(ndev, false);
|
||||
if (err) {
|
||||
netdev_err(ndev, "mana_detach failed: %d\n", err);
|
||||
apc->priv_flags = old_priv_flags;
|
||||
goto out;
|
||||
}
|
||||
|
||||
err = mana_attach(ndev);
|
||||
if (err) {
|
||||
netdev_err(ndev, "mana_attach failed: %d\n", err);
|
||||
apc->priv_flags = old_priv_flags;
|
||||
}
|
||||
}
|
||||
|
||||
out:
|
||||
mana_pre_dealloc_rxbufs(apc);
|
||||
clear_flag:
|
||||
mutex_lock(&apc->vport_mutex);
|
||||
apc->channel_changing = false;
|
||||
mutex_unlock(&apc->vport_mutex);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
const struct ethtool_ops mana_ethtool_ops = {
|
||||
.supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES |
|
||||
ETHTOOL_COALESCE_RX_USECS |
|
||||
|
|
@ -749,6 +857,7 @@ const struct ethtool_ops mana_ethtool_ops = {
|
|||
ETHTOOL_COALESCE_USE_ADAPTIVE_TX,
|
||||
.op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS |
|
||||
ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM |
|
||||
ETHTOOL_OP_NEEDS_RTNL_SPFLAGS |
|
||||
ETHTOOL_OP_NEEDS_RTNL_GLINK,
|
||||
.get_ethtool_stats = mana_get_ethtool_stats,
|
||||
.get_sset_count = mana_get_sset_count,
|
||||
|
|
@ -766,4 +875,6 @@ const struct ethtool_ops mana_ethtool_ops = {
|
|||
.set_ringparam = mana_set_ringparam,
|
||||
.get_link_ksettings = mana_get_link_ksettings,
|
||||
.get_link = ethtool_op_get_link,
|
||||
.get_priv_flags = mana_get_priv_flags,
|
||||
.set_priv_flags = mana_set_priv_flags,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -31,6 +31,12 @@ enum TRI_STATE {
|
|||
TRI_STATE_TRUE = 1
|
||||
};
|
||||
|
||||
/* MANA ethtool private flag bit positions */
|
||||
enum mana_priv_flag_bits {
|
||||
MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF = 0,
|
||||
MANA_PRIV_FLAG_MAX,
|
||||
};
|
||||
|
||||
/* Number of entries for hardware indirection table must be in power of 2 */
|
||||
#define MANA_INDIRECT_TABLE_MAX_SIZE 512
|
||||
#define MANA_INDIRECT_TABLE_DEF_SIZE 64
|
||||
|
|
@ -568,6 +574,8 @@ struct mana_port_context {
|
|||
u32 rxbpre_headroom;
|
||||
u32 rxbpre_frag_count;
|
||||
|
||||
u32 priv_flags;
|
||||
|
||||
struct bpf_prog *bpf_prog;
|
||||
|
||||
/* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user