github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-28 10:09:10 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'net-mlx5e-shampo-allow-high-order-pages-in-zerocopy-mode'

Browse Source 
Tariq Toukan says:

====================
net/mlx5e: SHAMPO, Allow high order pages in zerocopy mode

This series adds support for high order pages when io_uring/devmem
zero copy is used.

See detailed description by Dragos below.

The first patches are moving code around to allow using queue specific
parameters that are not just for XSK. They are a bit large as they touch
a lot of functions.

The middle part of the series is updating various formulas to remove
remaining hardcoded use of PAGE_SIZE/PAGE_SHIFT.

The last part adds support for high order pages by implementing the
queue configuration functions and allowing larger rx_page_size
configurations when in zero-copy mode.

Results show an increase in BW and a decrease in CPU usage.
The benchmark was done with the zcrx samples from liburing [0].

rx_buf_len=4K, oncpu [1]:
packets=3358832 (MB=820027), rps=55794 (MB/s=13621)
Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:       9    1.56    0.00   18.09   13.42    0.00   66.80    0.00    0.00    0.00    0.12

rx_buf_len=128K, oncpu [2]:
packets=3781376 (MB=923187), rps=62813 (MB/s=15335)
Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:       9    0.33    0.00    7.61   18.86    0.00   73.08    0.00    0.00    0.00    0.12

rx_buf_len=4K, offcpu [3]:
packets=3460368 (MB=844816), rps=57481 (MB/s=14033)
Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:       9    0.00    0.00    0.26    0.00    0.00   92.63    0.00    0.00    0.00    7.11
Average:      11    3.04    0.00   68.09   28.87    0.00    0.00    0.00    0.00    0.00    0.00

rx_buf_len=128K, offcpu [4]:
packets=4119840 (MB=1005820), rps=68435 (MB/s=16707)
Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:       9    0.00    0.00    0.87    0.00    0.00   63.77    0.00    0.00    0.00   35.36
Average:      11    1.96    0.00   43.68   54.37    0.00    0.00    0.00    0.00    0.00    0.00

[0] https://github.com/isilence/liburing/tree/zcrx/rx-buf-len

[1] commands:
  $> taskset -c 9 ./zcrx 6 -i eth2 -q 9 -A 1 -B 4096 -S 33554432
  $> ./send-zerocopy tcp -6 -D 2001:db8::1 -t 60 -C 0 -l 1 -b 1 -n 1 -z 1 -d -s 256000

[2] commands:
  $> taskset -c 9 ./zcrx 6 -i eth2 -q 9 -A 1 -B 131072 -S 33554432
  $> ./send-zerocopy tcp -6 -D 2001:db8::1 -t 60 -C 0 -l 1 -b 1 -n 1 -z 1 -d -s 256000

[3] commands:
  $> taskset -c 11 ./zcrx 6 -i eth2 -q 9 -A 1 -B 4096 -S 33554432
  $> ./send-zerocopy tcp -6 -D 2001:db8::1 -t 60 -C 0 -l 1 -b 1 -n 1 -z 1 -d -s 256000

[4] commands:
  $> taskset -c 11 ./zcrx 6 -i eth2 -q 9 -A 1 -B 131072 -S 33554432
  $> ./send-zerocopy tcp -6 -D 2001:db8::1 -t 60 -C 0 -l 1 -b 1 -n 1 -z 1 -d -s 256000
====================

Link: https://patch.msgid.link/20260223204155.1783580-1-tariqt@nvidia.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni 2026-02-26 10:54:41 +01:00
commit 90fcb0f3bc
11 changed files with 501 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
drivers/net/ethernet/mellanox/mlx5/core/en.h
View File

@ -80,6 +80,7 @@ struct page_pool;
#define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
#define MLX5E_PAGECNT_BIAS_MAX U16_MAX
#define MLX5E_RX_MAX_HEAD (256)
#define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8)
#define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE \
@ -1060,13 +1061,15 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv);
struct mlx5e_xsk_param;
struct mlx5e_rq_param;
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
struct mlx5e_rq_opt_param;
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param,
struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter,
struct mlx5e_rq *rq);
#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
void mlx5e_close_rq(struct mlx5e_rq *rq);
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter);
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param,
u16 q_counter);
void mlx5e_destroy_rq(struct mlx5e_rq *rq);
bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,

413
drivers/net/ethernet/mellanox/mlx5/core/en/params.c
View File

@ -10,6 +10,7 @@
#include <linux/dim.h>
#include <net/page_pool/types.h>
#include <net/xdp_sock_drv.h>
#include <net/netdev_queues.h>
#define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18
#define MLX5_REP_MPWRQ_MAX_LOG_WQE_SZ 17
@ -21,10 +22,20 @@ static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev)
return min_page_shift ? : 12;
}
u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev,
struct mlx5e_rq_opt_param *rqo)
{
u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT;
struct netdev_queue_config *qcfg = rqo ? rqo->qcfg : NULL;
struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev);
u8 req_page_shift;
if (xsk)
req_page_shift = order_base_2(xsk->chunk_size);
else if (qcfg && qcfg->rx_page_size)
req_page_shift = order_base_2(qcfg->rx_page_size);
else
req_page_shift = PAGE_SHIFT;
/* Regular RQ uses order-0 pages, the NIC must be able to map them. */
if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift))
@ -34,7 +45,8 @@ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xs
}
enum mlx5e_mpwrq_umr_mode
mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev,
struct mlx5e_rq_opt_param *rqo)
{
/* Different memory management schemes use different mechanisms to map
* user-mode memory. The stricter guarantees we have, the faster
@ -45,7 +57,8 @@ mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
* 3. KLM - indirect mapping to another MKey to arbitrary addresses, and
* mappings can have different sizes.
*/
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
bool unaligned = xsk ? xsk->unaligned : false;
bool oversized = false;
@ -225,12 +238,12 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
}
u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
u16 headroom;
if (xsk)
return xsk->headroom;
if (mlx5e_rqo_xsk_param(rqo))
return rqo->xsk->headroom;
headroom = NET_IP_ALIGN;
if (params->xdp_prog)
@ -263,19 +276,23 @@ static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool no_head_
static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_rq_opt_param *rqo,
bool mpwqe)
{
struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
bool no_head_tail_room;
u32 sz;
/* XSK frames are mapped as individual pages, because frames may come in
* an arbitrary order from random locations in the UMEM.
*/
if (xsk)
return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE;
if (xsk) {
return mpwqe ?
BIT(mlx5e_mpwrq_page_shift(mdev, rqo)) : PAGE_SIZE;
}
no_head_tail_room = params->xdp_prog && mpwqe && !mlx5e_rx_is_linear_skb(mdev, params, xsk);
no_head_tail_room = params->xdp_prog && mpwqe &&
!mlx5e_rx_is_linear_skb(mdev, params, rqo);
/* When no_head_tail_room is set, headroom and tailroom are excluded from skb calculations.
* no_head_tail_room should be set in the case of XDP with Striding RQ
@ -291,11 +308,12 @@ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev,
static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
u32 linear_stride_sz =
mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, true);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
order_base_2(linear_stride_sz);
@ -303,8 +321,10 @@ static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE)
return false;
@ -315,7 +335,7 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
* Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data
* must fit into a CPU page.
*/
if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE)
if (mlx5e_rx_get_linear_sz_skb(params, !!xsk) > PAGE_SIZE)
return false;
/* XSK frames must be big enough to hold the packet data. */
@ -349,12 +369,14 @@ static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
u8 log_wqe_num_of_strides =
mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo);
u8 log_wqe_stride_size =
mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
return mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
log_wqe_num_of_strides,
@ -363,18 +385,20 @@ bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
u32 linear_stride_sz =
mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, true);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
u8 log_num_strides;
u8 log_stride_sz;
u8 log_wqe_sz;
if (!mlx5e_rx_is_linear_skb(mdev, params, xsk))
if (!mlx5e_rx_is_linear_skb(mdev, params, rqo))
return false;
log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
log_stride_sz = order_base_2(linear_stride_sz);
log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
if (log_wqe_sz < log_stride_sz)
@ -389,13 +413,13 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
u8 log_pkts_per_wqe, page_shift, max_log_rq_size;
log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk);
page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, rqo);
page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode);
/* Numbers are unsigned, don't subtract to avoid underflow. */
@ -423,10 +447,11 @@ static u8 mlx5e_shampo_get_log_pkt_per_rsrv(struct mlx5e_params *params)
u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo))
return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params,
rqo, true));
/* XDP in mlx5e doesn't support multiple packets per page. */
if (params->xdp_prog)
@ -437,17 +462,18 @@ u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
u8 log_wqe_size, log_stride_size;
log_wqe_size = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo);
WARN(log_wqe_size < log_stride_size,
"Log WQE size %u < log stride size %u (page shift %u, umr mode %d, xsk on? %d)\n",
log_wqe_size, log_stride_size, page_shift, umr_mode, !!xsk);
log_wqe_size, log_stride_size, page_shift, umr_mode,
rqo && rqo->xsk);
return log_wqe_size - log_stride_size;
}
@ -459,14 +485,14 @@ u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz)
u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, xsk);
u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, rqo);
if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC)
return linear_headroom;
if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo))
return linear_headroom;
if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
@ -475,6 +501,18 @@ u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
return 0;
}
u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev)
{
if (mlx5_core_is_ecpf(mdev))
return PAGE_SIZE;
/* Two MTTs are needed to form an octword. Driver is using a
* single page per MTT for simplicity. Hence the limit of having
* at least 2 pages per WQE.
*/
return BIT(MLX5_MPWRQ_MAX_LOG_WQE_SZ - 1);
}
u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
{
bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE);
@ -535,10 +573,11 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params
}
int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
u16 max_mtu_pkts;
if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode)) {
@ -547,7 +586,7 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa
return -EOPNOTSUPP;
}
if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) {
if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo)) {
mlx5_core_err(mdev, "Striding RQ linear mode for XSK can't be activated with current params\n");
return -EINVAL;
}
@ -559,7 +598,8 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa
mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, xsk->unaligned));
if (params->log_rq_mtu_frames > max_mtu_pkts) {
mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n",
1 << params->log_rq_mtu_frames, xsk->chunk_size);
1 << params->log_rq_mtu_frames,
xsk->chunk_size);
return -EINVAL;
}
@ -672,7 +712,7 @@ static void mlx5e_rx_compute_wqe_bulk_params(struct mlx5e_params *params,
static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_rq_opt_param *rqo,
struct mlx5e_rq_frags_info *info,
u32 *xdp_frag_size)
{
@ -684,10 +724,11 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
int max_mtu;
int i;
if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
if (mlx5e_rx_is_linear_skb(mdev, params, rqo)) {
int frag_stride;
frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, false);
frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, rqo,
false);
info->arr[0].frag_size = byte_count;
info->arr[0].frag_stride = frag_stride;
@ -703,7 +744,7 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
goto out;
}
headroom = mlx5e_get_linear_rq_headroom(params, xsk);
headroom = mlx5e_get_linear_rq_headroom(params, rqo);
first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom);
max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size_max, frag_size_max,
@ -819,12 +860,13 @@ static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev,
static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo);
u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params,
rqo));
int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(params));
int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo));
int wqe_size = BIT(log_stride_sz) * num_strides;
int rsrv_size = MLX5E_SHAMPO_WQ_RESRV_SIZE;
@ -836,7 +878,7 @@ static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev,
static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_rq_opt_param *rqo,
struct mlx5e_cq_param *param)
{
bool hw_stridx = false;
@ -847,10 +889,13 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index);
if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk);
log_cq_size =
mlx5e_shampo_get_log_cq_size(mdev, params, rqo);
else
log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) +
mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
log_cq_size =
mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo) +
mlx5e_mpwqe_get_log_num_strides(mdev, params,
rqo);
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
log_cq_size = params->log_rq_mtu_frames;
@ -880,69 +925,90 @@ static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *param
MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN;
}
static int mlx5e_mpwqe_build_rq_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_rq_opt_param *rqo,
struct mlx5e_rq_param *rq_param)
{
u8 log_rq_sz = mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
u8 log_wqe_num_of_strides, log_wqe_stride_size;
enum mlx5e_mpwrq_umr_mode umr_mode;
void *rqc = rq_param->rqc;
u32 lro_timeout;
void *wq;
log_wqe_num_of_strides =
mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo);
log_wqe_stride_size =
mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo);
umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
wq = MLX5_ADDR_OF(rqc, rqc, wq);
if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
log_wqe_num_of_strides,
page_shift, umr_mode)) {
mlx5_core_err(mdev,
"Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n",
log_wqe_stride_size, log_wqe_num_of_strides,
umr_mode);
return -EINVAL;
}
MLX5_SET(wq, wq, log_wqe_num_of_strides,
log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
MLX5_SET(wq, wq, log_wqe_stride_size,
log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
MLX5_SET(wq, wq, log_wq_sz, log_rq_sz);
if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO)
return 0;
MLX5_SET(wq, wq, shampo_enable, true);
MLX5_SET(wq, wq, log_reservation_size,
MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE -
MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT);
MLX5_SET(wq, wq, log_max_num_of_packets_per_reservation,
mlx5e_shampo_get_log_pkt_per_rsrv(params));
MLX5_SET(wq, wq, log_headers_entry_size,
MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE -
MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT);
lro_timeout = mlx5e_choose_lro_timeout(mdev,
MLX5E_DEFAULT_SHAMPO_TIMEOUT);
MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout);
MLX5_SET(rqc, rqc, shampo_match_criteria_type,
MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED);
MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity,
MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE);
return 0;
}
int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_rq_param *param)
struct mlx5e_rq_opt_param *rqo,
struct mlx5e_rq_param *rq_param)
{
void *rqc = param->rqc;
void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
u32 lro_timeout;
void *rqc = rq_param->rqc;
int ndsegs = 1;
void *wq;
int err;
wq = MLX5_ADDR_OF(rqc, rqc, wq);
switch (params->rq_wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: {
u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
log_wqe_num_of_strides,
page_shift, umr_mode)) {
mlx5_core_err(mdev,
"Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n",
log_wqe_stride_size, log_wqe_num_of_strides,
umr_mode);
return -EINVAL;
}
MLX5_SET(wq, wq, log_wqe_num_of_strides,
log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
MLX5_SET(wq, wq, log_wqe_stride_size,
log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO)
break;
MLX5_SET(wq, wq, shampo_enable, true);
MLX5_SET(wq, wq, log_reservation_size,
MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE -
MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT);
MLX5_SET(wq, wq,
log_max_num_of_packets_per_reservation,
mlx5e_shampo_get_log_pkt_per_rsrv(params));
MLX5_SET(wq, wq, log_headers_entry_size,
MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE -
MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT);
lro_timeout =
mlx5e_choose_lro_timeout(mdev,
MLX5E_DEFAULT_SHAMPO_TIMEOUT);
MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout);
MLX5_SET(rqc, rqc, shampo_match_criteria_type,
MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED);
MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity,
MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE);
break;
}
default: /* MLX5_WQ_TYPE_CYCLIC */
MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
err = mlx5e_build_rq_frags_info(mdev, params, xsk, &param->frags_info,
&param->xdp_frag_size);
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
err = mlx5e_mpwqe_build_rq_param(mdev, params, rqo, rq_param);
if (err)
return err;
ndsegs = param->frags_info.num_frags;
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
err = mlx5e_build_rq_frags_info(mdev, params, rqo,
&rq_param->frags_info,
&rq_param->xdp_frag_size);
if (err)
return err;
ndsegs = rq_param->frags_info.num_frags;
}
MLX5_SET(wq, wq, wq_type, params->rq_wq_type);
@ -953,23 +1019,23 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable);
MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en);
param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
mlx5e_build_rx_cq_param(mdev, params, xsk, &param->cqp);
rq_param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
mlx5e_build_rx_cq_param(mdev, params, rqo, &rq_param->cqp);
return 0;
}
void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
struct mlx5e_rq_param *param)
struct mlx5e_rq_param *rq_param)
{
void *rqc = param->rqc;
void *rqc = rq_param->rqc;
void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
MLX5_SET(wq, wq, log_wq_stride,
mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1));
param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
rq_param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
}
void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev,
@ -1084,29 +1150,30 @@ u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
u8 umr_wqebbs;
umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
return umr_wqebbs *
(1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo));
}
static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_rq_param *rqp)
static u32 mlx5e_max_xsk_wqebbs(struct mlx5_core_dev *mdev,
struct mlx5e_params *params)
{
u32 wqebbs, total_pages, useful_space;
struct mlx5e_rq_opt_param rqo = {0};
struct mlx5e_xsk_param xsk = {0};
u32 max_xsk_wqebbs = 0;
u8 frame_shift;
/* MLX5_WQ_TYPE_CYCLIC */
if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
if (!params->xdp_prog)
return 0;
/* UMR WQEs for the regular RQ. */
wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL);
rqo.xsk = &xsk;
/* If XDP program is attached, XSK may be turned on at any time without
* restarting the channel. ICOSQ must be big enough to fit UMR WQEs of
@ -1118,41 +1185,54 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
* from capabilities. Hence, we have to try all valid values of XSK
* frame size (and page_shift) to find the maximum.
*/
if (params->xdp_prog) {
u32 max_xsk_wqebbs = 0;
u8 frame_shift;
for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT;
frame_shift <= PAGE_SHIFT; frame_shift++) {
u32 total_wqebbs;
for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT;
frame_shift <= PAGE_SHIFT; frame_shift++) {
/* The headroom doesn't affect the calculation. */
struct mlx5e_xsk_param xsk = {
.chunk_size = 1 << frame_shift,
.unaligned = false,
};
/* The headroom doesn't affect the calculations below. */
/* XSK aligned mode. */
max_xsk_wqebbs = max(max_xsk_wqebbs,
mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
/* XSK aligned mode. */
xsk.chunk_size = 1 << frame_shift;
xsk.unaligned = false;
total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo);
max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs);
/* XSK unaligned mode, frame size is a power of two. */
xsk.unaligned = true;
max_xsk_wqebbs = max(max_xsk_wqebbs,
mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
/* XSK unaligned mode, frame size is a power of two. */
xsk.unaligned = true;
total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo);
max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs);
/* XSK unaligned mode, frame size is not equal to stride size. */
xsk.chunk_size -= 1;
max_xsk_wqebbs = max(max_xsk_wqebbs,
mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
/* XSK unaligned mode, frame size is not equal to stride
* size.
*/
xsk.chunk_size -= 1;
total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo);
max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs);
/* XSK unaligned mode, frame size is a triple power of two. */
xsk.chunk_size = (1 << frame_shift) / 4 * 3;
max_xsk_wqebbs = max(max_xsk_wqebbs,
mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
}
wqebbs += max_xsk_wqebbs;
/* XSK unaligned mode, frame size is a triple power of two. */
xsk.chunk_size = (1 << frame_shift) / 4 * 3;
total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo);
max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs);
}
return max_xsk_wqebbs;
}
static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_rq_param *rq_param)
{
u32 wqebbs, total_pages, useful_space;
/* MLX5_WQ_TYPE_CYCLIC */
if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
/* UMR WQEs for the regular RQ. */
wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL);
wqebbs += mlx5e_max_xsk_wqebbs(mdev, params);
/* UMR WQEs don't cross the page boundary, they are padded with NOPs.
* This padding is always smaller than the max WQE size. That gives us
* at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes
@ -1222,12 +1302,15 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct netdev_queue_config *qcfg,
struct mlx5e_channel_param *cparam)
{
u8 icosq_log_wq_sz, async_icosq_log_wq_sz;
int err;
err = mlx5e_build_rq_param(mdev, params, NULL, &cparam->rq);
cparam->rq_opt.qcfg = qcfg;
err = mlx5e_build_rq_param(mdev, params, &cparam->rq_opt, &cparam->rq);
if (err)
return err;
@ -1241,3 +1324,13 @@ int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
return 0;
}
void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_channel_param *cparam)
{
cparam->rq_opt.xsk = xsk;
mlx5e_build_rq_param(mdev, params, &cparam->rq_opt, &cparam->rq);
mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
}

45
drivers/net/ethernet/mellanox/mlx5/core/en/params.h
View File

@ -12,6 +12,11 @@ struct mlx5e_xsk_param {
bool unaligned;
};
struct mlx5e_rq_opt_param {
struct mlx5e_xsk_param *xsk;
struct netdev_queue_config *qcfg;
};
struct mlx5e_cq_param {
u32 cqc[MLX5_ST_SZ_DW(cqc)];
struct mlx5_wq_param wq;
@ -38,6 +43,7 @@ struct mlx5e_sq_param {
struct mlx5e_channel_param {
struct mlx5e_rq_param rq;
struct mlx5e_rq_opt_param rq_opt;
struct mlx5e_sq_param txq_sq;
struct mlx5e_sq_param xdp_sq;
struct mlx5e_sq_param icosq;
@ -56,9 +62,11 @@ struct mlx5e_create_sq_param {
/* Striding RQ dynamic parameters */
u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev,
struct mlx5e_rq_opt_param *rqo);
enum mlx5e_mpwrq_umr_mode
mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev,
struct mlx5e_rq_opt_param *rqo);
u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode);
u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
enum mlx5e_mpwrq_umr_mode umr_mode);
@ -80,22 +88,22 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
bool slow_pci_heuristic(struct mlx5_core_dev *mdev);
int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
u32 mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_rq_param *rq_param);
@ -105,21 +113,22 @@ u32 mlx5e_shampo_hd_per_wq(struct mlx5_core_dev *mdev,
u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout);
u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz);
u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev);
/* Build queue parameters */
void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c);
int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_rq_opt_param *rqo,
struct mlx5e_rq_param *param);
void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev,
struct mlx5e_rq_param *param);
@ -136,13 +145,19 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
struct mlx5e_sq_param *param);
int mlx5e_build_channel_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct netdev_queue_config *qcfg,
struct mlx5e_channel_param *cparam);
void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_channel_param *cparam);
u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk);
struct mlx5e_rq_opt_param *rqo);
static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev,
struct mlx5e_params *params)
@ -158,4 +173,10 @@ static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev,
"enhanced" : "basic");
};
static inline struct mlx5e_xsk_param *
mlx5e_rqo_xsk_param(struct mlx5e_rq_opt_param *rqo)
{
return rqo ? rqo->xsk : NULL;
}
#endif /* __MLX5_EN_PARAMS_H__ */

4
drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
View File

@ -660,13 +660,13 @@ static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev,
struct net_device *netdev,
struct mlx5e_ptp_params *ptp_params)
{
struct mlx5e_rq_param *rq_params = &ptp_params->rq_param;
struct mlx5e_rq_param *rq_param = &ptp_params->rq_param;
struct mlx5e_params *params = &ptp_params->params;
params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC;
mlx5e_init_rq_type_params(mdev, params);
params->sw_mtu = netdev->max_mtu;
mlx5e_build_rq_param(mdev, params, NULL, rq_params);
mlx5e_build_rq_param(mdev, params, NULL, rq_param);
}
static void mlx5e_ptp_build_params(struct mlx5e_ptp *c,

5
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
View File

@ -37,9 +37,10 @@
#include <linux/bitfield.h>
#include <net/page_pool/helpers.h>
int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk)
int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
struct mlx5e_rq_opt_param *rqo)
{
int hr = mlx5e_get_linear_rq_headroom(params, xsk);
int hr = mlx5e_get_linear_rq_headroom(params, rqo);
/* Let S := SKB_DATA_ALIGN(sizeof(struct skb_shared_info)).
* The condition checked in mlx5e_rx_is_linear_skb is:

3
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
View File

@ -96,7 +96,8 @@ union mlx5e_xdp_info {
};
struct mlx5e_xsk_param;
int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);
int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
struct mlx5e_rq_opt_param *rqo);
bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx);
void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);

23
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
View File

@ -79,6 +79,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
struct xsk_buff_pool *pool, u16 ix)
{
struct mlx5e_params *params = &priv->channels.params;
struct mlx5e_channel_param *cparam;
enum mlx5e_mpwrq_umr_mode umr_mode;
struct mlx5e_xsk_param xsk;
struct mlx5e_channel *c;
int err;
@ -89,18 +91,24 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
return -EINVAL;
cparam = kvzalloc_obj(*cparam, GFP_KERNEL);
if (!cparam)
return -ENOMEM;
err = mlx5e_xsk_map_pool(mlx5_sd_ch_ix_get_dev(priv->mdev, ix), pool);
if (unlikely(err))
return err;
goto err_free_cparam;
err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix);
if (unlikely(err))
goto err_unmap_pool;
mlx5e_build_xsk_param(pool, &xsk);
mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam);
umr_mode = mlx5e_mpwrq_umr_mode(priv->mdev, &cparam->rq_opt);
if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
const char *recommendation = is_power_of_2(xsk.chunk_size) ?
"Upgrade firmware" : "Disable striding RQ";
@ -122,7 +130,7 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
c = priv->channels.c[ix];
err = mlx5e_open_xsk(priv, params, &xsk, pool, c);
err = mlx5e_open_xsk(priv, params, cparam, pool, c);
if (unlikely(err))
goto err_remove_pool;
@ -138,6 +146,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
mlx5e_deactivate_rq(&c->rq);
mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY);
kvfree(cparam);
return 0;
err_remove_pool:
@ -146,17 +156,22 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
err_unmap_pool:
mlx5e_xsk_unmap_pool(priv, pool);
err_free_cparam:
kvfree(cparam);
return err;
validate_closed:
/* Check the configuration in advance, rather than fail at a later stage
* (in mlx5e_xdp_set or on open) and end up with no channels.
*/
if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev)) {
err = -EINVAL;
goto err_remove_pool;
}
kvfree(cparam);
return 0;
}

59
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
View File

@ -9,9 +9,9 @@
static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk)
struct mlx5e_rq_opt_param *rqo)
{
if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
if (!mlx5e_rx_is_linear_skb(mdev, params, rqo)) {
mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n");
return -EINVAL;
}
@ -25,9 +25,14 @@ static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev,
#define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE)
bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_rq_opt_param *rqo,
struct mlx5_core_dev *mdev)
{
struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo);
if (WARN_ON(!xsk))
return false;
/* AF_XDP doesn't support frames larger than PAGE_SIZE,
* and xsk->chunk_size is limited to 65535 bytes.
*/
@ -42,21 +47,12 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
*/
switch (params->rq_wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk);
return !mlx5e_mpwrq_validate_xsk(mdev, params, rqo);
default: /* MLX5_WQ_TYPE_CYCLIC */
return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk);
return !mlx5e_legacy_rq_validate_xsk(mdev, params, rqo);
}
}
static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_channel_param *cparam)
{
mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq);
mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq);
}
static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct xsk_buff_pool *pool,
@ -90,19 +86,23 @@ static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id);
}
static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool,
struct mlx5e_xsk_param *xsk)
static int mlx5e_open_xsk_rq(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_channel_param *cparam,
struct xsk_buff_pool *pool)
{
struct mlx5e_rq_param *rq_param = &cparam->rq;
struct mlx5e_rq_opt_param *rqo = &cparam->rq_opt;
u16 q_counter = c->priv->q_counter[c->sd_ix];
struct mlx5e_rq *xskrq = &c->xskrq;
int err;
err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq);
err = mlx5e_init_xsk_rq(c, params, pool, rqo->xsk, xskrq);
if (err)
return err;
err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq);
err = mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu),
q_counter, xskrq);
if (err)
return err;
@ -111,30 +111,24 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param
}
int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
struct mlx5e_channel_param *cparam,
struct xsk_buff_pool *pool,
struct mlx5e_channel *c)
{
struct mlx5e_channel_param *cparam;
struct mlx5e_create_cq_param ccp;
int err;
mlx5e_build_create_cq_param(&ccp, c);
if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev))
return -EINVAL;
cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL);
if (!cparam)
return -ENOMEM;
mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam);
err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
&c->xskrq.cq);
if (unlikely(err))
goto err_free_cparam;
return err;
err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk);
err = mlx5e_open_xsk_rq(c, params, cparam, pool);
if (unlikely(err))
goto err_close_rx_cq;
@ -153,8 +147,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
if (unlikely(err))
goto err_close_tx_cq;
kvfree(cparam);
set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
return 0;
@ -168,9 +160,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
err_close_rx_cq:
mlx5e_close_cq(&c->xskrq.cq);
err_free_cparam:
kvfree(cparam);
return err;
}

6
drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h
View File

@ -9,10 +9,12 @@
struct mlx5e_xsk_param;
bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_rq_opt_param *rqo,
struct mlx5_core_dev *mdev);
struct mlx5e_channel_param;
int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool,
struct mlx5e_channel_param *cparam,
struct xsk_buff_pool *pool,
struct mlx5e_channel *c);
void mlx5e_close_xsk(struct mlx5e_channel *c);
void mlx5e_activate_xsk(struct mlx5e_channel *c);

194
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
View File

@ -636,14 +636,18 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work)
static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
{
rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
/* xsk can have page_shift < PAGE_SHIFT */
u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0);
u32 page_size = BIT(PAGE_SHIFT + page_order);
rq->wqe_overflow.page = alloc_pages(GFP_KERNEL, page_order);
if (!rq->wqe_overflow.page)
return -ENOMEM;
rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
PAGE_SIZE, rq->buff.map_dir);
page_size, rq->buff.map_dir);
if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
__free_page(rq->wqe_overflow.page);
__free_pages(rq->wqe_overflow.page, page_order);
return -ENOMEM;
}
return 0;
@ -651,9 +655,12 @@ static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
{
dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
rq->buff.map_dir);
__free_page(rq->wqe_overflow.page);
u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0);
u32 page_size = BIT(PAGE_SHIFT + page_order);
dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, page_size,
rq->buff.map_dir);
__free_pages(rq->wqe_overflow.page, page_order);
}
static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
@ -780,7 +787,7 @@ static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev,
static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
struct mlx5e_params *params,
struct mlx5e_rq_param *rqp,
struct mlx5e_rq_param *rq_param,
struct mlx5e_rq *rq,
int node)
{
@ -791,7 +798,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
return 0;
hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp);
hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rq_param);
hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE);
nentries = hd_buf_size / PAGE_SIZE;
if (!nentries) {
@ -851,19 +858,19 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
}
static int mlx5e_alloc_rq(struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_rq_param *rqp,
struct mlx5e_rq_param *rq_param,
struct mlx5e_rq_opt_param *rqo,
int node, struct mlx5e_rq *rq)
{
void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
struct mlx5_core_dev *mdev = rq->mdev;
void *rqc = rqp->rqc;
void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
u32 pool_order = 0;
u32 pool_size;
int wq_sz;
int err;
int i;
rqp->wq.db_numa_node = node;
rq_param->wq.db_numa_node = node;
INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
INIT_WORK(&rq->rx_timeout_work, mlx5e_rq_timeout_work);
@ -872,28 +879,28 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, rqo);
pool_size = 1 << params->log_rq_mtu_frames;
rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
switch (rq->wq_type) {
case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
&rq->wq_ctrl);
err = mlx5_wq_ll_create(mdev, &rq_param->wq, rqc_wq,
&rq->mpwqe.wq, &rq->wq_ctrl);
if (err)
goto err_rq_xdp_prog;
err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
if (err)
goto err_rq_wq_destroy;
rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, rqo);
err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
if (err)
goto err_rq_wq_destroy;
rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo);
rq->mpwqe.pages_per_wqe =
mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift,
rq->mpwqe.umr_mode);
@ -905,14 +912,18 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
rq->mpwqe.umr_mode);
pool_size = rq->mpwqe.pages_per_wqe <<
mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo);
pool_order = rq->mpwqe.page_shift - PAGE_SHIFT;
if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog)
if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo) &&
params->xdp_prog)
pool_size *= 2; /* additional page per packet for the linear part */
rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
rq->mpwqe.log_stride_sz =
mlx5e_mpwqe_get_log_stride_size(mdev, params,
rqo);
rq->mpwqe.num_strides =
BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo));
rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz);
rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
@ -925,14 +936,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
if (err)
goto err_rq_mkey;
err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, node);
err = mlx5_rq_shampo_alloc(mdev, params, rq_param, rq, node);
if (err)
goto err_free_mpwqe_info;
break;
default: /* MLX5_WQ_TYPE_CYCLIC */
err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
&rq->wq_ctrl);
err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq,
&rq->wqe.wq, &rq->wq_ctrl);
if (err)
goto err_rq_xdp_prog;
@ -940,7 +951,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
rq->wqe.info = rqp->frags_info;
rq->wqe.info = rq_param->frags_info;
rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
err = mlx5e_init_wqe_alloc_info(rq, node);
@ -948,7 +959,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
goto err_rq_wq_destroy;
}
if (xsk) {
if (mlx5e_rqo_xsk_param(rqo)) {
err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
MEM_TYPE_XSK_BUFF_POOL, NULL);
if (err)
@ -958,7 +969,13 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
/* Create a page_pool and register it with rxq */
struct page_pool_params pp_params = { 0 };
pp_params.order = 0;
if (WARN_ON(BIT(PAGE_SHIFT + pool_order) / 64 >
MLX5E_PAGECNT_BIAS_MAX)) {
err = -E2BIG;
goto err_free_by_rq_type;
}
pp_params.order = pool_order;
pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
pp_params.pool_size = pool_size;
pp_params.nid = node;
@ -966,7 +983,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
pp_params.napi = rq->cq.napi;
pp_params.netdev = rq->netdev;
pp_params.dma_dir = rq->buff.map_dir;
pp_params.max_len = PAGE_SIZE;
pp_params.max_len = BIT(PAGE_SHIFT + pool_order);
pp_params.queue_idx = rq->ix;
/* Shampo header data split allow for unreadable netmem */
@ -1085,7 +1102,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
xdp_rxq_info_unreg(&rq->xdp_rxq);
}
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter)
int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param,
u16 q_counter)
{
struct mlx5_core_dev *mdev = rq->mdev;
u8 ts_format;
@ -1107,7 +1125,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_cou
rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
wq = MLX5_ADDR_OF(rqc, rqc, wq);
memcpy(rqc, param->rqc, sizeof(param->rqc));
memcpy(rqc, rq_param->rqc, sizeof(rq_param->rqc));
MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn);
MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
@ -1323,8 +1341,8 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
}
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param,
struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter,
struct mlx5e_rq *rq)
{
struct mlx5_core_dev *mdev = rq->mdev;
@ -1333,11 +1351,11 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
__set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state);
err = mlx5e_alloc_rq(params, xsk, param, node, rq);
err = mlx5e_alloc_rq(params, rq_param, rqo, node, rq);
if (err)
return err;
err = mlx5e_create_rq(rq, param, q_counter);
err = mlx5e_create_rq(rq, rq_param, q_counter);
if (err)
goto err_free_rq;
@ -2506,17 +2524,20 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
return err;
}
static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
struct mlx5e_rq_param *rq_params)
static int mlx5e_open_rxq_rq(struct mlx5e_channel *c,
struct mlx5e_params *params,
struct mlx5e_rq_param *rq_param,
struct mlx5e_rq_opt_param *rqo)
{
u16 q_counter = c->priv->q_counter[c->sd_ix];
int err;
err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq);
err = mlx5e_init_rxq_rq(c, params, rq_param->xdp_frag_size, &c->rq);
if (err)
return err;
return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq);
return mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu),
q_counter, &c->rq);
}
static struct mlx5e_icosq *
@ -2619,7 +2640,7 @@ static int mlx5e_open_queues(struct mlx5e_channel *c,
if (err)
goto err_close_icosq;
err = mlx5e_open_rxq_rq(c, params, &cparam->rq);
err = mlx5e_open_rxq_rq(c, params, &cparam->rq, &cparam->rq_opt);
if (err)
goto err_close_sqs;
@ -2764,6 +2785,7 @@ static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c)
static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
struct mlx5e_params *params,
struct netdev_queue_config *qcfg,
struct xsk_buff_pool *xsk_pool,
struct mlx5e_channel **cp)
{
@ -2797,7 +2819,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
goto err_free;
}
err = mlx5e_build_channel_param(mdev, params, cparam);
err = mlx5e_build_channel_param(mdev, params, qcfg, cparam);
if (err)
goto err_free;
@ -2828,7 +2850,8 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
if (xsk_pool) {
mlx5e_build_xsk_param(xsk_pool, &xsk);
err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam);
err = mlx5e_open_xsk(priv, params, cparam, xsk_pool, c);
if (unlikely(err))
goto err_close_queues;
}
@ -2921,7 +2944,8 @@ int mlx5e_open_channels(struct mlx5e_priv *priv,
if (chs->params.xdp_prog)
xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
err = mlx5e_open_channel(priv, i, &chs->params, xsk_pool, &chs->c[i]);
err = mlx5e_open_channel(priv, i, &chs->params, NULL,
xsk_pool, &chs->c[i]);
if (err)
goto err_close_channels;
}
@ -3577,15 +3601,14 @@ static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
struct mlx5e_rq *rq,
struct mlx5e_rq_param *param)
struct mlx5e_rq_param *rq_param)
{
void *rqc = param->rqc;
void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq);
int err;
param->wq.db_numa_node = param->wq.buf_numa_node;
rq_param->wq.db_numa_node = rq_param->wq.buf_numa_node;
err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, &rq->wqe.wq,
&rq->wq_ctrl);
if (err)
return err;
@ -4586,6 +4609,7 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
for (ix = 0; ix < chs->params.num_channels; ix++) {
struct xsk_buff_pool *xsk_pool =
mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix);
struct mlx5e_rq_opt_param rqo = {0};
struct mlx5e_xsk_param xsk;
int max_xdp_mtu;
@ -4593,12 +4617,13 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
continue;
mlx5e_build_xsk_param(xsk_pool, &xsk);
max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk);
rqo.xsk = &xsk;
max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &rqo);
/* Validate XSK params and XDP MTU in advance */
if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) ||
if (!mlx5e_validate_xsk_param(new_params, &rqo, mdev) ||
new_params->sw_mtu > max_xdp_mtu) {
u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
u32 hr = mlx5e_get_linear_rq_headroom(new_params, &rqo);
int max_mtu_frame, max_mtu_page, max_mtu;
/* Two criteria must be met:
@ -5549,9 +5574,52 @@ static const struct netdev_stat_ops mlx5e_stat_ops = {
struct mlx5_qmgmt_data {
struct mlx5e_channel *c;
struct mlx5e_channel_param cparam;
};
static void mlx5e_queue_default_qcfg(struct net_device *dev,
struct netdev_queue_config *qcfg)
{
qcfg->rx_page_size = PAGE_SIZE;
}
static int mlx5e_queue_validate_qcfg(struct net_device *dev,
struct netdev_queue_config *qcfg,
struct netlink_ext_ack *extack)
{
struct mlx5e_priv *priv = netdev_priv(dev);
struct mlx5_core_dev *mdev = priv->mdev;
u32 max;
if (!is_power_of_2(qcfg->rx_page_size)) {
netdev_err(priv->netdev, "rx_page_size not power of 2: %u",
qcfg->rx_page_size);
return -EINVAL;
}
max = mlx5e_mpwrq_max_page_size(mdev);
if (qcfg->rx_page_size < PAGE_SIZE || qcfg->rx_page_size > max) {
netdev_err(priv->netdev,
"Selected rx_page_size %u not in supported range [%lu, %u]\n",
qcfg->rx_page_size, PAGE_SIZE, max);
return -ERANGE;
}
return 0;
}
static bool mlx5e_queue_validate_page_size(struct net_device *dev,
struct netdev_queue_config *qcfg,
int queue_index)
{
if (qcfg->rx_page_size == PAGE_SIZE)
return true;
if (!netif_rxq_has_unreadable_mp(dev, queue_index))
return false;
return true;
}
static int mlx5e_queue_mem_alloc(struct net_device *dev,
struct netdev_queue_config *qcfg,
void *newq, int queue_index)
@ -5560,7 +5628,6 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev,
struct mlx5e_priv *priv = netdev_priv(dev);
struct mlx5e_channels *chs = &priv->channels;
struct mlx5e_params params = chs->params;
struct mlx5_core_dev *mdev;
int err;
mutex_lock(&priv->state_lock);
@ -5584,12 +5651,14 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev,
goto unlock;
}
mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, queue_index);
err = mlx5e_build_channel_param(mdev, &params, &new->cparam);
if (err)
if (!mlx5e_queue_validate_page_size(dev, qcfg, queue_index)) {
netdev_err(priv->netdev, "High order pages are supported only in Zero-Copy mode\n");
err = -EINVAL;
goto unlock;
}
err = mlx5e_open_channel(priv, queue_index, &params, NULL, &new->c);
err = mlx5e_open_channel(priv, queue_index, &params, qcfg, NULL,
&new->c);
unlock:
mutex_unlock(&priv->state_lock);
return err;
@ -5668,6 +5737,9 @@ static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = {
.ndo_queue_start = mlx5e_queue_start,
.ndo_queue_stop = mlx5e_queue_stop,
.ndo_queue_get_dma_dev = mlx5e_queue_get_dma_dev,
.ndo_default_qcfg = mlx5e_queue_default_qcfg,
.ndo_validate_qcfg = mlx5e_queue_validate_qcfg,
.supported_params = QCFG_RX_PAGE_SIZE,
};
static void mlx5e_build_nic_netdev(struct net_device *netdev)

36
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
View File

@ -272,8 +272,6 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem);
}
#define MLX5E_PAGECNT_BIAS_MAX (PAGE_SIZE / 64)
static int mlx5e_page_alloc_fragmented(struct page_pool *pp,
struct mlx5e_frag_page *frag_page)
{
@ -1847,11 +1845,14 @@ mlx5e_shampo_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq,
struct mlx5e_frag_page *frag_page,
u32 data_bcnt, u32 data_offset)
{
u32 page_size = BIT(rq->mpwqe.page_shift);
net_prefetchw(skb->data);
do {
/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt);
u32 pg_consumed_bytes = min_t(u32, page_size - data_offset,
data_bcnt);
unsigned int truesize = pg_consumed_bytes;
mlx5e_add_skb_frag(rq, skb, frag_page, data_offset,
@ -1872,6 +1873,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
struct mlx5e_frag_page *head_page = frag_page;
struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf;
u32 page_size = BIT(rq->mpwqe.page_shift);
u32 frag_offset = head_offset;
u32 byte_cnt = cqe_bcnt;
struct skb_shared_info *sinfo;
@ -1926,9 +1928,9 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
linear_hr = skb_headroom(skb);
linear_data_len = headlen;
linear_frame_sz = MLX5_SKB_FRAG_SZ(skb_end_offset(skb));
if (unlikely(frag_offset >= PAGE_SIZE)) {
if (unlikely(frag_offset >= page_size)) {
frag_page++;
frag_offset -= PAGE_SIZE;
frag_offset -= page_size;
}
}
@ -1940,7 +1942,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
while (byte_cnt) {
/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
pg_consumed_bytes =
min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
min_t(u32, page_size - frag_offset, byte_cnt);
if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
truesize += pg_consumed_bytes;
@ -1978,7 +1980,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
nr_frags_free = old_nr_frags - sinfo->nr_frags;
if (unlikely(nr_frags_free)) {
frag_page -= nr_frags_free;
truesize -= (nr_frags_free - 1) * PAGE_SIZE +
truesize -= (nr_frags_free - 1) * page_size +
ALIGN(pg_consumed_bytes,
BIT(rq->mpwqe.log_stride_sz));
}
@ -2166,15 +2168,16 @@ mlx5e_shampo_flush_skb(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match)
rq->hw_gro_data->skb = NULL;
}
static bool
mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt)
static bool mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb,
u16 data_bcnt,
u32 page_size)
{
int nr_frags = skb_shinfo(skb)->nr_frags;
if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE)
if (page_size >= GRO_LEGACY_MAX_SIZE)
return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE;
else
return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;
return page_size * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE;
}
static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
@ -2183,18 +2186,19 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
u16 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe);
u32 wqe_offset = be32_to_cpu(cqe->shampo.data_offset);
u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe);
u32 data_offset = wqe_offset & (PAGE_SIZE - 1);
u32 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
u16 wqe_id = be16_to_cpu(cqe->wqe_id);
u32 page_idx = wqe_offset >> PAGE_SHIFT;
u16 head_size = cqe->shampo.header_size;
struct sk_buff **skb = &rq->hw_gro_data->skb;
bool flush = cqe->shampo.flush;
bool match = cqe->shampo.match;
u32 page_size = BIT(rq->mpwqe.page_shift);
struct mlx5e_rq_stats *stats = rq->stats;
struct mlx5e_rx_wqe_ll *wqe;
struct mlx5e_mpw_info *wi;
struct mlx5_wq_ll *wq;
u32 data_offset;
u32 page_idx;
wi = mlx5e_get_mpw_info(rq, wqe_id);
wi->consumed_strides += cstrides;
@ -2210,7 +2214,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
goto mpwrq_cqe_out;
}
if (*skb && (!match || !(mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt)))) {
data_offset = wqe_offset & (page_size - 1);
page_idx = wqe_offset >> rq->mpwqe.page_shift;
if (*skb &&
!(match && mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt,
page_size))) {
match = false;
mlx5e_shampo_flush_skb(rq, cqe, match);
}