From 8e6f6e92d3fe7032371417d2036b1c6897d2c2e5 Mon Sep 17 00:00:00 2001 From: Yael Chemla Date: Sun, 16 Mar 2025 10:14:33 +0200 Subject: [PATCH 1/4] net/mlx5e: Ensure each counter group uses its PCAM bit The code was incorrectly relying on PCAM bit of ppcnt_statistical_group for accessing per_lane_error_counters. If ppcnt_statistical_group PCAM bit was not set, we would not read per_lane_error_counters, even when its PCAM bit is set. Given the existing device capabilities, it seems to cause no harm, so this change primarily serves as cleanup. Signed-off-by: Yael Chemla Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Reviewed-by: Jacob Keller Reviewed-by: Kalesh AP Link: https://patch.msgid.link/1742112876-2890-2-git-send-email-tariqt@nvidia.com Signed-off-by: Paolo Abeni --- .../ethernet/mellanox/mlx5/core/en_stats.c | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 611ec4b6f370..77d34037b92b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1272,11 +1272,9 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(phy) ethtool_puts(data, "link_down_events_phy"); - if (!MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) - return; - - for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++) - ethtool_puts(data, pport_phy_statistical_stats_desc[i].format); + if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) + for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++) + ethtool_puts(data, pport_phy_statistical_stats_desc[i].format); if (MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS; i++) @@ -1294,15 +1292,13 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(phy) data, MLX5_GET(ppcnt_reg, priv->stats.pport.phy_counters, counter_set.phys_layer_cntrs.link_down_events)); - if (!MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) - return; - - for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++) - mlx5e_ethtool_put_stat( - data, - MLX5E_READ_CTR64_BE( - &priv->stats.pport.phy_statistical_counters, - pport_phy_statistical_stats_desc, i)); + if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) + for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++) + mlx5e_ethtool_put_stat( + data, + MLX5E_READ_CTR64_BE( + &priv->stats.pport.phy_statistical_counters, + pport_phy_statistical_stats_desc, i)); if (MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS; i++) From da4fa5d8817dafb1a000f67b3fee5993000e8e06 Mon Sep 17 00:00:00 2001 From: Yael Chemla Date: Sun, 16 Mar 2025 10:14:34 +0200 Subject: [PATCH 2/4] net/mlx5e: Access PHY layer counter group as other counter groups Adjust the way physical layer counters group is accessed to match the generic method used for accessing other PPCNT counter groups. Signed-off-by: Yael Chemla Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Reviewed-by: Jacob Keller Link: https://patch.msgid.link/1742112876-2890-3-git-send-email-tariqt@nvidia.com Signed-off-by: Paolo Abeni --- .../ethernet/mellanox/mlx5/core/en_stats.c | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 77d34037b92b..0cf0c920532f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1227,6 +1227,13 @@ void mlx5e_stats_ts_get(struct mlx5e_priv *priv, mutex_unlock(&priv->state_lock); } +#define PPORT_PHY_LAYER_OFF(c) \ + MLX5_BYTE_OFF(ppcnt_reg, \ + counter_set.phys_layer_cntrs.c) +static const struct counter_desc pport_phy_layer_cntrs_stats_desc[] = { + { "link_down_events_phy", PPORT_PHY_LAYER_OFF(link_down_events) } +}; + #define PPORT_PHY_STATISTICAL_OFF(c) \ MLX5_BYTE_OFF(ppcnt_reg, \ counter_set.phys_layer_statistical_cntrs.c##_high) @@ -1243,6 +1250,8 @@ pport_phy_statistical_err_lanes_stats_desc[] = { { "rx_err_lane_3_phy", PPORT_PHY_STATISTICAL_OFF(phy_corrected_bits_lane3) }, }; +#define NUM_PPORT_PHY_LAYER_COUNTERS \ + ARRAY_SIZE(pport_phy_layer_cntrs_stats_desc) #define NUM_PPORT_PHY_STATISTICAL_COUNTERS \ ARRAY_SIZE(pport_phy_statistical_stats_desc) #define NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS \ @@ -1253,8 +1262,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(phy) struct mlx5_core_dev *mdev = priv->mdev; int num_stats; - /* "1" for link_down_events special counter */ - num_stats = 1; + num_stats = NUM_PPORT_PHY_LAYER_COUNTERS; num_stats += MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) ? NUM_PPORT_PHY_STATISTICAL_COUNTERS : 0; @@ -1270,7 +1278,8 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(phy) struct mlx5_core_dev *mdev = priv->mdev; int i; - ethtool_puts(data, "link_down_events_phy"); + for (i = 0; i < NUM_PPORT_PHY_LAYER_COUNTERS; i++) + ethtool_puts(data, pport_phy_layer_cntrs_stats_desc[i].format); if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++) @@ -1287,10 +1296,12 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(phy) struct mlx5_core_dev *mdev = priv->mdev; int i; - /* link_down_events_phy has special handling since it is not stored in __be64 format */ - mlx5e_ethtool_put_stat( - data, MLX5_GET(ppcnt_reg, priv->stats.pport.phy_counters, - counter_set.phys_layer_cntrs.link_down_events)); + for (i = 0; i < NUM_PPORT_PHY_LAYER_COUNTERS; i++) + mlx5e_ethtool_put_stat( + data, + MLX5E_READ_CTR32_BE(&priv->stats.pport + .phy_counters, + pport_phy_layer_cntrs_stats_desc, i)); if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++) From 4c737ceb690cd35a8caf909f2998ef79f15cc6a0 Mon Sep 17 00:00:00 2001 From: Yael Chemla Date: Sun, 16 Mar 2025 10:14:35 +0200 Subject: [PATCH 3/4] net/mlx5e: Get counter group size by FW capability Retrieve the number of fields supported by each PPCNT counter group based on the FW capability for this group. Signed-off-by: Yael Chemla Signed-off-by: Tariq Toukan Reviewed-by: Jacob Keller Reviewed-by: Kalesh AP Link: https://patch.msgid.link/1742112876-2890-4-git-send-email-tariqt@nvidia.com Signed-off-by: Paolo Abeni --- .../ethernet/mellanox/mlx5/core/en_stats.c | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 0cf0c920532f..a417962acfa9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1257,6 +1257,13 @@ pport_phy_statistical_err_lanes_stats_desc[] = { #define NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS \ ARRAY_SIZE(pport_phy_statistical_err_lanes_stats_desc) +#define NUM_PPORT_PHY_STATISTICAL_LOOPBACK_COUNTERS(dev) \ + (MLX5_CAP_PCAM_FEATURE(dev, ppcnt_statistical_group) ? \ + NUM_PPORT_PHY_STATISTICAL_COUNTERS : 0) +#define NUM_PPORT_PHY_STATISTICAL_PER_LANE_LOOPBACK_COUNTERS(dev) \ + (MLX5_CAP_PCAM_FEATURE(dev, per_lane_error_counters) ? \ + NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS : 0) + static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(phy) { struct mlx5_core_dev *mdev = priv->mdev; @@ -1264,11 +1271,9 @@ static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(phy) num_stats = NUM_PPORT_PHY_LAYER_COUNTERS; - num_stats += MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) ? - NUM_PPORT_PHY_STATISTICAL_COUNTERS : 0; + num_stats += NUM_PPORT_PHY_STATISTICAL_LOOPBACK_COUNTERS(mdev); - num_stats += MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters) ? - NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS : 0; + num_stats += NUM_PPORT_PHY_STATISTICAL_PER_LANE_LOOPBACK_COUNTERS(mdev); return num_stats; } @@ -1281,14 +1286,15 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(phy) for (i = 0; i < NUM_PPORT_PHY_LAYER_COUNTERS; i++) ethtool_puts(data, pport_phy_layer_cntrs_stats_desc[i].format); - if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) - for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++) - ethtool_puts(data, pport_phy_statistical_stats_desc[i].format); + for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_LOOPBACK_COUNTERS(mdev); i++) + ethtool_puts(data, pport_phy_statistical_stats_desc[i].format); - if (MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) - for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS; i++) - ethtool_puts(data, - pport_phy_statistical_err_lanes_stats_desc[i].format); + for (i = 0; + i < NUM_PPORT_PHY_STATISTICAL_PER_LANE_LOOPBACK_COUNTERS(mdev); + i++) + ethtool_puts(data, + pport_phy_statistical_err_lanes_stats_desc[i] + .format); } static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(phy) @@ -1303,23 +1309,21 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(phy) .phy_counters, pport_phy_layer_cntrs_stats_desc, i)); - if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) - for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_COUNTERS; i++) - mlx5e_ethtool_put_stat( - data, - MLX5E_READ_CTR64_BE( - &priv->stats.pport.phy_statistical_counters, - pport_phy_statistical_stats_desc, i)); + for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_LOOPBACK_COUNTERS(mdev); i++) + mlx5e_ethtool_put_stat( + data, + MLX5E_READ_CTR64_BE( + &priv->stats.pport.phy_statistical_counters, + pport_phy_statistical_stats_desc, i)); - if (MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) - for (i = 0; i < NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS; i++) - mlx5e_ethtool_put_stat( - data, - MLX5E_READ_CTR64_BE( - &priv->stats.pport - .phy_statistical_counters, - pport_phy_statistical_err_lanes_stats_desc, - i)); + for (i = 0; + i < NUM_PPORT_PHY_STATISTICAL_PER_LANE_LOOPBACK_COUNTERS(mdev); + i++) + mlx5e_ethtool_put_stat( + data, + MLX5E_READ_CTR64_BE( + &priv->stats.pport.phy_statistical_counters, + pport_phy_statistical_err_lanes_stats_desc, i)); } static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(phy) From c3b999cad7ec39a5487b20e8b6e737d2ab0c5393 Mon Sep 17 00:00:00 2001 From: Yael Chemla Date: Sun, 16 Mar 2025 10:14:36 +0200 Subject: [PATCH 4/4] net/mlx5e: Expose port reset cycle recovery counter via ethtool Display recovery event of PPCNT recovery counters group. Counts (per link) the number of total successful recovery events of any recovery types during port reset cycle. Signed-off-by: Yael Chemla Reviewed-by: Cosmin Ratiu Signed-off-by: Tariq Toukan Reviewed-by: Jacob Keller Link: https://patch.msgid.link/1742112876-2890-5-git-send-email-tariqt@nvidia.com Signed-off-by: Paolo Abeni --- .../ethernet/mellanox/mlx5/counters.rst | 5 +++ .../ethernet/mellanox/mlx5/core/en_stats.c | 44 ++++++++++++++++--- .../ethernet/mellanox/mlx5/core/en_stats.h | 4 ++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/Documentation/networking/device_drivers/ethernet/mellanox/mlx5/counters.rst b/Documentation/networking/device_drivers/ethernet/mellanox/mlx5/counters.rst index 99d95be4d159..43d72c8b713b 100644 --- a/Documentation/networking/device_drivers/ethernet/mellanox/mlx5/counters.rst +++ b/Documentation/networking/device_drivers/ethernet/mellanox/mlx5/counters.rst @@ -1082,6 +1082,11 @@ like flow control, FEC and more. need to replace the cable/transceiver. - Error + * - `total_success_recovery_phy` + - The number of total successful recovery events of any type during + ports reset cycle. + - Error + * - `rx_out_of_buffer` - Number of times receive queue had no software buffers allocated for the adapter's incoming traffic. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index a417962acfa9..acb00fd7efa4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1250,12 +1250,22 @@ pport_phy_statistical_err_lanes_stats_desc[] = { { "rx_err_lane_3_phy", PPORT_PHY_STATISTICAL_OFF(phy_corrected_bits_lane3) }, }; +#define PPORT_PHY_RECOVERY_OFF(c) \ + MLX5_BYTE_OFF(ppcnt_reg, counter_set.phys_layer_recovery_cntrs.c) +static const struct counter_desc +pport_phy_recovery_cntrs_stats_desc[] = { + { "total_success_recovery_phy", + PPORT_PHY_RECOVERY_OFF(total_successful_recovery_events) } +}; + #define NUM_PPORT_PHY_LAYER_COUNTERS \ ARRAY_SIZE(pport_phy_layer_cntrs_stats_desc) #define NUM_PPORT_PHY_STATISTICAL_COUNTERS \ ARRAY_SIZE(pport_phy_statistical_stats_desc) #define NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS \ ARRAY_SIZE(pport_phy_statistical_err_lanes_stats_desc) +#define NUM_PPORT_PHY_RECOVERY_COUNTERS \ + ARRAY_SIZE(pport_phy_recovery_cntrs_stats_desc) #define NUM_PPORT_PHY_STATISTICAL_LOOPBACK_COUNTERS(dev) \ (MLX5_CAP_PCAM_FEATURE(dev, ppcnt_statistical_group) ? \ @@ -1263,6 +1273,9 @@ pport_phy_statistical_err_lanes_stats_desc[] = { #define NUM_PPORT_PHY_STATISTICAL_PER_LANE_LOOPBACK_COUNTERS(dev) \ (MLX5_CAP_PCAM_FEATURE(dev, per_lane_error_counters) ? \ NUM_PPORT_PHY_STATISTICAL_PER_LANE_COUNTERS : 0) +#define NUM_PPORT_PHY_RECOVERY_LOOPBACK_COUNTERS(dev) \ + (MLX5_CAP_PCAM_FEATURE(dev, ppcnt_recovery_counters) ? \ + NUM_PPORT_PHY_RECOVERY_COUNTERS : 0) static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(phy) { @@ -1275,6 +1288,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(phy) num_stats += NUM_PPORT_PHY_STATISTICAL_PER_LANE_LOOPBACK_COUNTERS(mdev); + num_stats += NUM_PPORT_PHY_RECOVERY_LOOPBACK_COUNTERS(mdev); return num_stats; } @@ -1295,6 +1309,10 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(phy) ethtool_puts(data, pport_phy_statistical_err_lanes_stats_desc[i] .format); + + for (i = 0; i < NUM_PPORT_PHY_RECOVERY_LOOPBACK_COUNTERS(mdev); i++) + ethtool_puts(data, + pport_phy_recovery_cntrs_stats_desc[i].format); } static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(phy) @@ -1324,6 +1342,13 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(phy) MLX5E_READ_CTR64_BE( &priv->stats.pport.phy_statistical_counters, pport_phy_statistical_err_lanes_stats_desc, i)); + + for (i = 0; i < NUM_PPORT_PHY_RECOVERY_LOOPBACK_COUNTERS(mdev); i++) + mlx5e_ethtool_put_stat( + data, + MLX5E_READ_CTR32_BE( + &priv->stats.pport.phy_recovery_counters, + pport_phy_recovery_cntrs_stats_desc, i)); } static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(phy) @@ -1339,12 +1364,21 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(phy) MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP); mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); - if (!MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) - return; + if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group)) { + out = pstats->phy_statistical_counters; + MLX5_SET(ppcnt_reg, in, grp, + MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP); + mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, + 0); + } - out = pstats->phy_statistical_counters; - MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP); - mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); + if (MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_recovery_counters)) { + out = pstats->phy_recovery_counters; + MLX5_SET(ppcnt_reg, in, grp, + MLX5_PHYSICAL_LAYER_RECOVERY_GROUP); + mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, + 0); + } } void mlx5e_get_link_ext_stats(struct net_device *dev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index 5961c569cfe0..0d87947e348d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -309,6 +309,9 @@ struct mlx5e_vport_stats { #define PPORT_PHY_STATISTICAL_GET(pstats, c) \ MLX5_GET64(ppcnt_reg, (pstats)->phy_statistical_counters, \ counter_set.phys_layer_statistical_cntrs.c##_high) +#define PPORT_PHY_RECOVERY_GET(pstats, c) \ + MLX5_GET64(ppcnt_reg, (pstats)->phy_recovery_counters, \ + counter_set.phys_layer_recovery_cntrs.c) #define PPORT_PER_PRIO_GET(pstats, prio, c) \ MLX5_GET64(ppcnt_reg, pstats->per_prio_counters[prio], \ counter_set.eth_per_prio_grp_data_layout.c##_high) @@ -324,6 +327,7 @@ struct mlx5e_pport_stats { __be64 per_prio_counters[NUM_PPORT_PRIO][MLX5_ST_SZ_QW(ppcnt_reg)]; __be64 phy_counters[MLX5_ST_SZ_QW(ppcnt_reg)]; __be64 phy_statistical_counters[MLX5_ST_SZ_QW(ppcnt_reg)]; + __be64 phy_recovery_counters[MLX5_ST_SZ_QW(ppcnt_reg)]; __be64 eth_ext_counters[MLX5_ST_SZ_QW(ppcnt_reg)]; __be64 per_tc_prio_counters[NUM_PPORT_PRIO][MLX5_ST_SZ_QW(ppcnt_reg)]; __be64 per_tc_congest_prio_counters[NUM_PPORT_PRIO][MLX5_ST_SZ_QW(ppcnt_reg)];