Merge branch 'add-ethtool-coalesce_rx_cqe_frames-nsecs-and-use-it-in-mana-driver'

Haiyang Zhang says:

====================
add ethtool COALESCE_RX_CQE_FRAMES/NSECS and use it in MANA driver

Add two parameters for drivers supporting Rx CQE Coalescing.

ETHTOOL_A_COALESCE_RX_CQE_FRAMES:
Maximum number of frames that can be coalesced into a CQE or
writeback.

ETHTOOL_A_COALESCE_RX_CQE_NSECS:
Max time in nanoseconds after the first packet arrival in a
coalesced CQE or writeback to be sent.

Also implement it in MANA driver with the new parameter and
counters.
====================

Link: https://patch.msgid.link/20260317191826.1346111-1-haiyangz@linux.microsoft.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2026-03-18 20:01:14 -07:00
commit a7fb05cbb8
8 changed files with 183 additions and 38 deletions

View File

@ -857,6 +857,12 @@ attribute-sets:
name: tx-profile
type: nest
nested-attributes: profile
-
name: rx-cqe-frames
type: u32
-
name: rx-cqe-nsecs
type: u32
-
name: pause-stat
@ -2253,6 +2259,8 @@ operations:
- tx-aggr-time-usecs
- rx-profile
- tx-profile
- rx-cqe-frames
- rx-cqe-nsecs
dump: *coalesce-get-op
-
name: coalesce-set

View File

@ -1076,6 +1076,8 @@ Kernel response contents:
``ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS`` u32 time (us), aggr, Tx
``ETHTOOL_A_COALESCE_RX_PROFILE`` nested profile of DIM, Rx
``ETHTOOL_A_COALESCE_TX_PROFILE`` nested profile of DIM, Tx
``ETHTOOL_A_COALESCE_RX_CQE_FRAMES`` u32 max packets, Rx CQE
``ETHTOOL_A_COALESCE_RX_CQE_NSECS`` u32 delay (ns), Rx CQE
=========================================== ====== =======================
Attributes are only included in reply if their value is not zero or the
@ -1109,6 +1111,13 @@ well with frequent small-sized URBs transmissions.
to DIM parameters, see `Generic Network Dynamic Interrupt Moderation (Net DIM)
<https://www.kernel.org/doc/Documentation/networking/net_dim.rst>`_.
Rx CQE coalescing allows multiple received packets to be coalesced into a
single Completion Queue Entry (CQE) or descriptor writeback.
``ETHTOOL_A_COALESCE_RX_CQE_FRAMES`` describes the maximum number of
frames that can be coalesced into a CQE or writeback.
``ETHTOOL_A_COALESCE_RX_CQE_NSECS`` describes max time in nanoseconds after
the first packet arrival in a coalesced CQE or writeback to be sent.
COALESCE_SET
============
@ -1147,6 +1156,8 @@ Request contents:
``ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS`` u32 time (us), aggr, Tx
``ETHTOOL_A_COALESCE_RX_PROFILE`` nested profile of DIM, Rx
``ETHTOOL_A_COALESCE_TX_PROFILE`` nested profile of DIM, Tx
``ETHTOOL_A_COALESCE_RX_CQE_FRAMES`` u32 max packets, Rx CQE
``ETHTOOL_A_COALESCE_RX_CQE_NSECS`` u32 delay (ns), Rx CQE
=========================================== ====== =======================
Request is rejected if it attributes declared as unsupported by driver (i.e.

View File

@ -1365,6 +1365,7 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
sizeof(resp));
req->hdr.req.msg_version = GDMA_MESSAGE_V2;
req->hdr.resp.msg_version = GDMA_MESSAGE_V2;
req->vport = apc->port_handle;
req->num_indir_entries = apc->indir_table_sz;
@ -1376,7 +1377,9 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
req->update_hashkey = update_key;
req->update_indir_tab = update_tab;
req->default_rxobj = apc->default_rxobj;
req->cqe_coalescing_enable = 0;
if (rx != TRI_STATE_FALSE)
req->cqe_coalescing_enable = apc->cqe_coalescing_enable;
if (update_key)
memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
@ -1405,8 +1408,13 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
resp.hdr.status);
err = -EPROTO;
goto out;
}
if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2)
apc->cqe_coalescing_timeout_ns =
resp.cqe_coalescing_timeout_ns;
netdev_info(ndev, "Configured steering vPort %llu entries %u\n",
apc->port_handle, apc->indir_table_sz);
out:
@ -1915,11 +1923,12 @@ static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va,
}
static void mana_rx_skb(void *buf_va, bool from_pool,
struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq)
struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq,
int i)
{
struct mana_stats_rx *rx_stats = &rxq->stats;
struct net_device *ndev = rxq->ndev;
uint pkt_len = cqe->ppi[0].pkt_len;
uint pkt_len = cqe->ppi[i].pkt_len;
u16 rxq_idx = rxq->rxq_idx;
struct napi_struct *napi;
struct xdp_buff xdp = {};
@ -1963,7 +1972,7 @@ static void mana_rx_skb(void *buf_va, bool from_pool,
}
if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {
hash_value = cqe->ppi[0].pkt_hash;
hash_value = cqe->ppi[i].pkt_hash;
if (cqe->rx_hashtype & MANA_HASH_L4)
skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);
@ -2098,9 +2107,11 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
struct mana_recv_buf_oob *rxbuf_oob;
struct mana_port_context *apc;
struct device *dev = gc->dev;
bool coalesced = false;
void *old_buf = NULL;
u32 curr, pktlen;
bool old_fp;
int i;
apc = netdev_priv(ndev);
@ -2112,12 +2123,15 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
++ndev->stats.rx_dropped;
rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];
netdev_warn_once(ndev, "Dropped a truncated packet\n");
goto drop;
mana_move_wq_tail(rxq->gdma_rq,
rxbuf_oob->wqe_inf.wqe_size_in_bu);
mana_post_pkt_rxq(rxq);
return;
case CQE_RX_COALESCED_4:
netdev_err(ndev, "RX coalescing is unsupported\n");
apc->eth_stats.rx_coalesced_err++;
return;
coalesced = true;
break;
case CQE_RX_OBJECT_FENCE:
complete(&rxq->fence_event);
@ -2130,30 +2144,47 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
return;
}
pktlen = oob->ppi[0].pkt_len;
for (i = 0; i < MANA_RXCOMP_OOB_NUM_PPI; i++) {
old_buf = NULL;
pktlen = oob->ppi[i].pkt_len;
if (pktlen == 0)
break;
if (pktlen == 0) {
/* data packets should never have packetlength of zero */
netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
rxq->gdma_id, cq->gdma_id, rxq->rxobj);
return;
curr = rxq->buf_index;
rxbuf_oob = &rxq->rx_oobs[curr];
WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp);
/* Unsuccessful refill will have old_buf == NULL.
* In this case, mana_rx_skb() will drop the packet.
*/
mana_rx_skb(old_buf, old_fp, oob, rxq, i);
mana_move_wq_tail(rxq->gdma_rq,
rxbuf_oob->wqe_inf.wqe_size_in_bu);
mana_post_pkt_rxq(rxq);
if (!coalesced)
break;
}
curr = rxq->buf_index;
rxbuf_oob = &rxq->rx_oobs[curr];
WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp);
/* Unsuccessful refill will have old_buf == NULL.
* In this case, mana_rx_skb() will drop the packet.
/* Collect coalesced CQE count based on packets processed.
* Coalesced CQEs have at least 2 packets, so index is i - 2.
*/
mana_rx_skb(old_buf, old_fp, oob, rxq);
drop:
mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
mana_post_pkt_rxq(rxq);
if (i > 1) {
u64_stats_update_begin(&rxq->stats.syncp);
rxq->stats.coalesced_cqe[i - 2]++;
u64_stats_update_end(&rxq->stats.syncp);
} else if (!i && !pktlen) {
u64_stats_update_begin(&rxq->stats.syncp);
rxq->stats.pkt_len0_err++;
u64_stats_update_end(&rxq->stats.syncp);
netdev_err_once(ndev,
"RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
rxq->gdma_id, cq->gdma_id, rxq->rxobj);
}
}
static void mana_poll_rx_cq(struct mana_cq *cq)
@ -3332,6 +3363,7 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
apc->port_handle = INVALID_MANA_HANDLE;
apc->pf_filter_handle = INVALID_MANA_HANDLE;
apc->port_idx = port_idx;
apc->cqe_coalescing_enable = 0;
mutex_init(&apc->vport_mutex);
apc->vport_use_count = 0;

View File

@ -20,8 +20,6 @@ static const struct mana_stats_desc mana_eth_stats[] = {
tx_cqe_unknown_type)},
{"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats,
tx_linear_pkt_cnt)},
{"rx_coalesced_err", offsetof(struct mana_ethtool_stats,
rx_coalesced_err)},
{"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats,
rx_cqe_unknown_type)},
};
@ -151,7 +149,7 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
{
struct mana_port_context *apc = netdev_priv(ndev);
unsigned int num_queues = apc->num_queues;
int i;
int i, j;
if (stringset != ETH_SS_STATS)
return;
@ -170,6 +168,9 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
ethtool_sprintf(&data, "rx_%d_xdp_drop", i);
ethtool_sprintf(&data, "rx_%d_xdp_tx", i);
ethtool_sprintf(&data, "rx_%d_xdp_redirect", i);
ethtool_sprintf(&data, "rx_%d_pkt_len0_err", i);
for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
ethtool_sprintf(&data, "rx_%d_coalesced_cqe_%d", i, j + 2);
}
for (i = 0; i < num_queues; i++) {
@ -203,6 +204,8 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
u64 xdp_xmit;
u64 xdp_drop;
u64 xdp_tx;
u64 pkt_len0_err;
u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
u64 tso_packets;
u64 tso_bytes;
u64 tso_inner_packets;
@ -211,7 +214,7 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
u64 short_pkt_fmt;
u64 csum_partial;
u64 mana_map_err;
int q, i = 0;
int q, i = 0, j;
if (!apc->port_is_up)
return;
@ -241,6 +244,9 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
xdp_drop = rx_stats->xdp_drop;
xdp_tx = rx_stats->xdp_tx;
xdp_redirect = rx_stats->xdp_redirect;
pkt_len0_err = rx_stats->pkt_len0_err;
for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
coalesced_cqe[j] = rx_stats->coalesced_cqe[j];
} while (u64_stats_fetch_retry(&rx_stats->syncp, start));
data[i++] = packets;
@ -248,6 +254,9 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
data[i++] = xdp_drop;
data[i++] = xdp_tx;
data[i++] = xdp_redirect;
data[i++] = pkt_len0_err;
for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
data[i++] = coalesced_cqe[j];
}
for (q = 0; q < num_queues; q++) {
@ -390,6 +399,61 @@ static void mana_get_channels(struct net_device *ndev,
channel->combined_count = apc->num_queues;
}
#define MANA_RX_CQE_NSEC_DEF 2048
static int mana_get_coalesce(struct net_device *ndev,
struct ethtool_coalesce *ec,
struct kernel_ethtool_coalesce *kernel_coal,
struct netlink_ext_ack *extack)
{
struct mana_port_context *apc = netdev_priv(ndev);
kernel_coal->rx_cqe_frames =
apc->cqe_coalescing_enable ? MANA_RXCOMP_OOB_NUM_PPI : 1;
kernel_coal->rx_cqe_nsecs = apc->cqe_coalescing_timeout_ns;
/* Return the default timeout value for old FW not providing
* this value.
*/
if (apc->port_is_up && apc->cqe_coalescing_enable &&
!kernel_coal->rx_cqe_nsecs)
kernel_coal->rx_cqe_nsecs = MANA_RX_CQE_NSEC_DEF;
return 0;
}
static int mana_set_coalesce(struct net_device *ndev,
struct ethtool_coalesce *ec,
struct kernel_ethtool_coalesce *kernel_coal,
struct netlink_ext_ack *extack)
{
struct mana_port_context *apc = netdev_priv(ndev);
u8 saved_cqe_coalescing_enable;
int err;
if (kernel_coal->rx_cqe_frames != 1 &&
kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI) {
NL_SET_ERR_MSG_FMT(extack,
"rx-frames must be 1 or %u, got %u",
MANA_RXCOMP_OOB_NUM_PPI,
kernel_coal->rx_cqe_frames);
return -EINVAL;
}
saved_cqe_coalescing_enable = apc->cqe_coalescing_enable;
apc->cqe_coalescing_enable =
kernel_coal->rx_cqe_frames == MANA_RXCOMP_OOB_NUM_PPI;
if (!apc->port_is_up)
return 0;
err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
if (err)
apc->cqe_coalescing_enable = saved_cqe_coalescing_enable;
return err;
}
static int mana_set_channels(struct net_device *ndev,
struct ethtool_channels *channels)
{
@ -510,6 +574,7 @@ static int mana_get_link_ksettings(struct net_device *ndev,
}
const struct ethtool_ops mana_ethtool_ops = {
.supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES,
.get_ethtool_stats = mana_get_ethtool_stats,
.get_sset_count = mana_get_sset_count,
.get_strings = mana_get_strings,
@ -520,6 +585,8 @@ const struct ethtool_ops mana_ethtool_ops = {
.set_rxfh = mana_set_rxfh,
.get_channels = mana_get_channels,
.set_channels = mana_set_channels,
.get_coalesce = mana_get_coalesce,
.set_coalesce = mana_set_coalesce,
.get_ringparam = mana_get_ringparam,
.set_ringparam = mana_set_ringparam,
.get_link_ksettings = mana_get_link_ksettings,

View File

@ -332,6 +332,8 @@ struct kernel_ethtool_coalesce {
u32 tx_aggr_max_bytes;
u32 tx_aggr_max_frames;
u32 tx_aggr_time_usecs;
u32 rx_cqe_frames;
u32 rx_cqe_nsecs;
};
/**
@ -380,7 +382,9 @@ bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32,
#define ETHTOOL_COALESCE_TX_AGGR_TIME_USECS BIT(26)
#define ETHTOOL_COALESCE_RX_PROFILE BIT(27)
#define ETHTOOL_COALESCE_TX_PROFILE BIT(28)
#define ETHTOOL_COALESCE_ALL_PARAMS GENMASK(28, 0)
#define ETHTOOL_COALESCE_RX_CQE_FRAMES BIT(29)
#define ETHTOOL_COALESCE_RX_CQE_NSECS BIT(30)
#define ETHTOOL_COALESCE_ALL_PARAMS GENMASK(30, 0)
#define ETHTOOL_COALESCE_USECS \
(ETHTOOL_COALESCE_RX_USECS | ETHTOOL_COALESCE_TX_USECS)

View File

@ -61,8 +61,11 @@ enum TRI_STATE {
#define MAX_PORTS_IN_MANA_DEV 256
/* Maximum number of packets per coalesced CQE */
#define MANA_RXCOMP_OOB_NUM_PPI 4
/* Update this count whenever the respective structures are changed */
#define MANA_STATS_RX_COUNT 5
#define MANA_STATS_RX_COUNT (6 + MANA_RXCOMP_OOB_NUM_PPI - 1)
#define MANA_STATS_TX_COUNT 11
#define MANA_RX_FRAG_ALIGNMENT 64
@ -73,6 +76,8 @@ struct mana_stats_rx {
u64 xdp_drop;
u64 xdp_tx;
u64 xdp_redirect;
u64 pkt_len0_err;
u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
struct u64_stats_sync syncp;
};
@ -227,8 +232,6 @@ struct mana_rxcomp_perpkt_info {
u32 pkt_hash;
}; /* HW DATA */
#define MANA_RXCOMP_OOB_NUM_PPI 4
/* Receive completion OOB */
struct mana_rxcomp_oob {
struct mana_cqe_header cqe_hdr;
@ -378,7 +381,6 @@ struct mana_ethtool_stats {
u64 tx_cqe_err;
u64 tx_cqe_unknown_type;
u64 tx_linear_pkt_cnt;
u64 rx_coalesced_err;
u64 rx_cqe_unknown_type;
};
@ -557,6 +559,9 @@ struct mana_port_context {
bool port_is_up;
bool port_st_save; /* Saved port state */
u8 cqe_coalescing_enable;
u32 cqe_coalescing_timeout_ns;
struct mana_ethtool_stats eth_stats;
struct mana_ethtool_phy_stats phy_stats;
@ -902,6 +907,10 @@ struct mana_cfg_rx_steer_req_v2 {
struct mana_cfg_rx_steer_resp {
struct gdma_resp_hdr hdr;
/* V2 */
u32 cqe_coalescing_timeout_ns;
u32 reserved1;
}; /* HW DATA */
/* Register HW vPort */

View File

@ -371,6 +371,8 @@ enum {
ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS,
ETHTOOL_A_COALESCE_RX_PROFILE,
ETHTOOL_A_COALESCE_TX_PROFILE,
ETHTOOL_A_COALESCE_RX_CQE_FRAMES,
ETHTOOL_A_COALESCE_RX_CQE_NSECS,
__ETHTOOL_A_COALESCE_CNT,
ETHTOOL_A_COALESCE_MAX = (__ETHTOOL_A_COALESCE_CNT - 1)

View File

@ -118,6 +118,8 @@ static int coalesce_reply_size(const struct ethnl_req_info *req_base,
nla_total_size(sizeof(u32)) + /* _TX_AGGR_MAX_BYTES */
nla_total_size(sizeof(u32)) + /* _TX_AGGR_MAX_FRAMES */
nla_total_size(sizeof(u32)) + /* _TX_AGGR_TIME_USECS */
nla_total_size(sizeof(u32)) + /* _RX_CQE_FRAMES */
nla_total_size(sizeof(u32)) + /* _RX_CQE_NSECS */
total_modersz * 2; /* _{R,T}X_PROFILE */
}
@ -269,7 +271,11 @@ static int coalesce_fill_reply(struct sk_buff *skb,
coalesce_put_u32(skb, ETHTOOL_A_COALESCE_TX_AGGR_MAX_FRAMES,
kcoal->tx_aggr_max_frames, supported) ||
coalesce_put_u32(skb, ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS,
kcoal->tx_aggr_time_usecs, supported))
kcoal->tx_aggr_time_usecs, supported) ||
coalesce_put_u32(skb, ETHTOOL_A_COALESCE_RX_CQE_FRAMES,
kcoal->rx_cqe_frames, supported) ||
coalesce_put_u32(skb, ETHTOOL_A_COALESCE_RX_CQE_NSECS,
kcoal->rx_cqe_nsecs, supported))
return -EMSGSIZE;
if (!req_base->dev || !req_base->dev->irq_moder)
@ -338,6 +344,8 @@ const struct nla_policy ethnl_coalesce_set_policy[] = {
[ETHTOOL_A_COALESCE_TX_AGGR_MAX_BYTES] = { .type = NLA_U32 },
[ETHTOOL_A_COALESCE_TX_AGGR_MAX_FRAMES] = { .type = NLA_U32 },
[ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS] = { .type = NLA_U32 },
[ETHTOOL_A_COALESCE_RX_CQE_FRAMES] = { .type = NLA_U32 },
[ETHTOOL_A_COALESCE_RX_CQE_NSECS] = { .type = NLA_U32 },
[ETHTOOL_A_COALESCE_RX_PROFILE] =
NLA_POLICY_NESTED(coalesce_profile_policy),
[ETHTOOL_A_COALESCE_TX_PROFILE] =
@ -570,6 +578,10 @@ __ethnl_set_coalesce(struct ethnl_req_info *req_info, struct genl_info *info,
tb[ETHTOOL_A_COALESCE_TX_AGGR_MAX_FRAMES], &mod);
ethnl_update_u32(&kernel_coalesce.tx_aggr_time_usecs,
tb[ETHTOOL_A_COALESCE_TX_AGGR_TIME_USECS], &mod);
ethnl_update_u32(&kernel_coalesce.rx_cqe_frames,
tb[ETHTOOL_A_COALESCE_RX_CQE_FRAMES], &mod);
ethnl_update_u32(&kernel_coalesce.rx_cqe_nsecs,
tb[ETHTOOL_A_COALESCE_RX_CQE_NSECS], &mod);
if (dev->irq_moder && dev->irq_moder->profile_flags & DIM_PROFILE_RX) {
ret = ethnl_update_profile(dev, &dev->irq_moder->rx_profile,