diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c index 8d9a3b5ec973..a3382f6a6b74 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c @@ -168,8 +168,6 @@ int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, return err; } -#define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2 - u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels) { u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE, @@ -179,13 +177,6 @@ u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels) return min_t(u32, rqt_size, max_cap_rqt_size); } -#define MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH 256 - -unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void) -{ - return MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH / MLX5E_UNIFORM_SPREAD_RQT_FACTOR; -} - void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt) { mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h index 2f9e04a8418f..c6d0a92b132c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h @@ -7,6 +7,7 @@ #include #define MLX5E_INDIR_MIN_RQT_SIZE (BIT(8)) +#define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 4 struct mlx5_core_dev; @@ -38,7 +39,6 @@ static inline u32 mlx5e_rqt_get_rqtn(struct mlx5e_rqt *rqt) } u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels); -unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void); int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id); int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c index a2ec67a122d9..de435df7ca50 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. +#include #include "rss.h" #define mlx5e_rss_warn(__dev, format, ...) \ @@ -85,9 +86,33 @@ bool mlx5e_rss_get_inner_ft_support(struct mlx5e_rss *rss) return rss->params.inner_ft_support; } -void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels) +u32 *mlx5e_rss_get_indir_table(struct mlx5e_rss *rss) { - rss->indir.actual_table_size = mlx5e_rqt_size(rss->mdev, num_channels); + return rss->indir.table; +} + +void mlx5e_rss_set_indir_actual_size(struct mlx5e_rss *rss, u32 size) +{ + rss->indir.actual_table_size = size; +} + +/* Handles non-default contexts, replicate existing pattern into new entries, + * matching what ethtool_rxfh_ctxs_resize() does. + */ +void mlx5e_rss_ctx_resize(struct mlx5e_rss *rss, u32 new_size) +{ + u32 old_size = rss->indir.actual_table_size; + u32 i; + + for (i = old_size; i < new_size; i++) + rss->indir.table[i] = rss->indir.table[i % old_size]; +} + +void mlx5e_rss_indir_resize(struct mlx5e_rss *rss, struct net_device *netdev, + u32 new_size) +{ + ethtool_rxfh_indir_resize(netdev, rss->indir.table, + rss->indir.actual_table_size, new_size); } int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h index 17664757a561..1bb0434612a4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h @@ -34,7 +34,9 @@ struct mlx5e_rss; int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir, u32 actual_table_size, u32 max_table_size); void mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir *indir); -void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels); +void mlx5e_rss_ctx_resize(struct mlx5e_rss *rss, u32 new_size); +void mlx5e_rss_indir_resize(struct mlx5e_rss *rss, struct net_device *netdev, + u32 new_size); struct mlx5e_rss * mlx5e_rss_init(struct mlx5_core_dev *mdev, const struct mlx5e_rss_params *params, @@ -46,6 +48,8 @@ void mlx5e_rss_refcnt_dec(struct mlx5e_rss *rss); unsigned int mlx5e_rss_refcnt_read(struct mlx5e_rss *rss); bool mlx5e_rss_get_inner_ft_support(struct mlx5e_rss *rss); +u32 *mlx5e_rss_get_indir_table(struct mlx5e_rss *rss); +void mlx5e_rss_set_indir_actual_size(struct mlx5e_rss *rss, u32 size); u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool inner); bool mlx5e_rss_valid_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool inner); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c index 92974b11ec75..e940635f5dcb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c @@ -40,13 +40,38 @@ static u32 *get_vhca_ids(struct mlx5e_rx_res *res, int offset) return multi_vhca ? res->rss_vhca_ids + offset : NULL; } -void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch) +/* Updates the indirection table SW shadow, does not update the HW resources yet + */ +void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch, + struct net_device *netdev) { + u32 new_size = mlx5e_rqt_size(res->mdev, nch); int i; - for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { - if (res->rss[i]) - mlx5e_rss_params_indir_modify_actual_size(res->rss[i], nch); + WARN_ON_ONCE(res->rss_active); + + /* Default context: fold/unfold user-configured table, then update size + * and reset to uniform when unconfigured. + */ + mlx5e_rss_indir_resize(res->rss[0], netdev, new_size); + + /* mlx5e_rss_indir_resize() is a no-op when the table is not + * user-configured. actual_table_size is updated after the resize + * because ethtool_rxfh_indir_resize() uses it as the old size to + * replicate the pattern; updating it first would make the grow a no-op. + * It must be updated before mlx5e_rss_set_indir_uniform() so that + * the uniform fill covers all new entries, not just the old ones. + */ + mlx5e_rss_set_indir_actual_size(res->rss[0], new_size); + if (!netif_is_rxfh_configured(netdev)) + mlx5e_rss_set_indir_uniform(res->rss[0], nch); + + /* Non-default contexts */ + for (i = 1; i < MLX5E_MAX_NUM_RSS; i++) { + if (res->rss[i]) { + mlx5e_rss_ctx_resize(res->rss[i], new_size); + mlx5e_rss_set_indir_actual_size(res->rss[i], new_size); + } } } @@ -209,13 +234,6 @@ static void mlx5e_rx_res_rss_disable(struct mlx5e_rx_res *res) } } -/* Updates the indirection table SW shadow, does not update the HW resources yet */ -void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch) -{ - WARN_ON_ONCE(res->rss_active); - mlx5e_rss_set_indir_uniform(res->rss[0], nch); -} - void mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, u32 *indir, u8 *key, u8 *hfunc, bool *symmetric) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h index 675780120a20..8fff18d64978 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h @@ -48,7 +48,6 @@ void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *ch unsigned int ix, bool xsk); /* Configuration API */ -void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch); void mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, u32 *indir, u8 *key, u8 *hfunc, bool *symmetric); @@ -68,7 +67,8 @@ int mlx5e_rx_res_rss_destroy(struct mlx5e_rx_res *res, u32 rss_idx); int mlx5e_rx_res_rss_cnt(struct mlx5e_rx_res *res); int mlx5e_rx_res_rss_index(struct mlx5e_rx_res *res, struct mlx5e_rss *rss); struct mlx5e_rss *mlx5e_rx_res_rss_get(struct mlx5e_rx_res *res, u32 rss_idx); -void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch); +void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch, + struct net_device *netdev); /* Workaround for hairpin */ struct mlx5e_rss_params_hash mlx5e_rx_res_get_current_hash(struct mlx5e_rx_res *res); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index bb61e2179078..300d1cb2e070 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -45,6 +45,10 @@ #define LANES_UNKNOWN 0 +#define MLX5E_MAX_INDIR_RQT_SIZE \ + roundup_pow_of_two(MLX5E_MAX_NUM_CHANNELS * \ + MLX5E_UNIFORM_SPREAD_RQT_FACTOR) + void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv, struct ethtool_drvinfo *drvinfo) { @@ -495,11 +499,16 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, { struct mlx5e_params *cur_params = &priv->channels.params; unsigned int count = ch->combined_count; + int new_rqt_size, cur_rqt_size; struct mlx5e_params new_params; + struct mlx5e_rss *rss0; bool arfs_enabled; + bool has_rss_ctxs; bool opened; int err = 0; + ASSERT_RTNL(); + if (!count) { netdev_info(priv->netdev, "%s: combined_count=0 not supported\n", __func__); @@ -509,34 +518,37 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, if (cur_params->num_channels == count) return 0; - mutex_lock(&priv->state_lock); - - if (mlx5e_rx_res_get_current_hash(priv->rx_res).hfunc == ETH_RSS_HASH_XOR) { - unsigned int xor8_max_channels = mlx5e_rqt_max_num_channels_allowed_for_xor8(); - - if (count > xor8_max_channels) { - err = -EINVAL; - netdev_err(priv->netdev, "%s: Requested number of channels (%d) exceeds the maximum allowed by the XOR8 RSS hfunc (%d)\n", - __func__, count, xor8_max_channels); - goto out; - } + new_rqt_size = mlx5e_rqt_size(priv->mdev, count); + /* Validate that all non-default RSS contexts can be resized before + * committing to the channel count change. + * ethtool_rxfh_ctxs_can_resize() acquires rss_lock internally and + * cannot be called under state_lock (rss_lock -> state_lock ordering). + */ + has_rss_ctxs = priv->rx_res && mlx5e_rx_res_rss_cnt(priv->rx_res) > 1; + if (has_rss_ctxs) { + err = ethtool_rxfh_ctxs_can_resize(priv->netdev, new_rqt_size); + if (err) + return err; } - /* If RXFH is configured, changing the channels number is allowed only if - * it does not require resizing the RSS table. This is because the previous - * configuration may no longer be compatible with the new RSS table. - */ - if (netif_is_rxfh_configured(priv->netdev)) { - int cur_rqt_size = mlx5e_rqt_size(priv->mdev, cur_params->num_channels); - int new_rqt_size = mlx5e_rqt_size(priv->mdev, count); + mutex_lock(&priv->state_lock); - if (new_rqt_size != cur_rqt_size) { - err = -EINVAL; - netdev_err(priv->netdev, - "%s: RXFH is configured, block changing channels number that affects RSS table size (new: %d, current: %d)\n", - __func__, new_rqt_size, cur_rqt_size); - goto out; - } + if (!priv->rx_res) { + err = -EINVAL; + goto out; + } + + cur_rqt_size = mlx5e_rqt_size(priv->mdev, cur_params->num_channels); + rss0 = mlx5e_rx_res_rss_get(priv->rx_res, 0); + + if (!ethtool_rxfh_indir_can_resize(priv->netdev, + mlx5e_rss_get_indir_table(rss0), + cur_rqt_size, new_rqt_size)) { + netdev_err(priv->netdev, + "%s: cannot resize RSS table (%u -> %u); reset indirection table to allow this change\n", + __func__, cur_rqt_size, new_rqt_size); + err = -EINVAL; + goto out; } /* Don't allow changing the number of channels if HTB offload is active, @@ -584,6 +596,14 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, out: mutex_unlock(&priv->state_lock); + /* After a successful channel count change that altered the RQT size, + * fold or unfold the indirection tables of all non-default RSS + * contexts. Must run after state_lock is released because + * ethtool_rxfh_ctxs_resize() acquires rss_lock internally. + */ + if (!err && cur_rqt_size != new_rqt_size && has_rss_ctxs) + ethtool_rxfh_ctxs_resize(priv->netdev, new_rqt_size); + return err; } @@ -1501,29 +1521,6 @@ static int mlx5e_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param * return 0; } -static int mlx5e_rxfh_hfunc_check(struct mlx5e_priv *priv, - const struct ethtool_rxfh_param *rxfh, - struct netlink_ext_ack *extack) -{ - unsigned int count; - - count = priv->channels.params.num_channels; - - if (rxfh->hfunc == ETH_RSS_HASH_XOR) { - unsigned int xor8_max_channels = mlx5e_rqt_max_num_channels_allowed_for_xor8(); - - if (count > xor8_max_channels) { - NL_SET_ERR_MSG_FMT_MOD( - extack, - "Number of channels (%u) exceeds the max for XOR8 RSS (%u)", - count, xor8_max_channels); - return -EINVAL; - } - } - - return 0; -} - static int mlx5e_set_rxfh(struct net_device *dev, struct ethtool_rxfh_param *rxfh, struct netlink_ext_ack *extack) @@ -1535,16 +1532,11 @@ static int mlx5e_set_rxfh(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); - if (err) - goto unlock; - err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, rxfh->rss_context, rxfh->indir, rxfh->key, hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc, rxfh->input_xfrm == RXH_XFRM_NO_CHANGE ? NULL : &symmetric); -unlock: mutex_unlock(&priv->state_lock); return err; } @@ -1561,10 +1553,6 @@ static int mlx5e_create_rxfh_context(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); - if (err) - goto unlock; - err = mlx5e_rx_res_rss_init(priv->rx_res, rxfh->rss_context, priv->channels.params.num_channels); if (err) @@ -1601,16 +1589,11 @@ static int mlx5e_modify_rxfh_context(struct net_device *dev, mutex_lock(&priv->state_lock); - err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack); - if (err) - goto unlock; - err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, rxfh->rss_context, rxfh->indir, rxfh->key, hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc, rxfh->input_xfrm == RXH_XFRM_NO_CHANGE ? NULL : &symmetric); -unlock: mutex_unlock(&priv->state_lock); return err; } @@ -2740,6 +2723,7 @@ const struct ethtool_ops mlx5e_ethtool_ops = { .supported_input_xfrm = RXH_XFRM_SYM_OR_XOR, .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT | ETHTOOL_RING_USE_HDS_THRS, + .rxfh_indir_space = MLX5E_MAX_INDIR_RQT_SIZE, .get_drvinfo = mlx5e_get_drvinfo, .get_link = ethtool_op_get_link, .get_link_ext_state = mlx5e_get_link_ext_state, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 85b1ccbd351f..a904e468c197 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3296,12 +3296,9 @@ static int mlx5e_num_channels_changed(struct mlx5e_priv *priv) } /* This function may be called on attach, before priv->rx_res is created. */ - if (priv->rx_res) { - mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count); - - if (!netif_is_rxfh_configured(priv->netdev)) - mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count); - } + if (priv->rx_res) + mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count, + netdev); return 0; }