diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index ea2cd1f5d1d0..c7ac6ebe8290 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -80,6 +80,7 @@ struct page_pool; #define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \ SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) +#define MLX5E_PAGECNT_BIAS_MAX U16_MAX #define MLX5E_RX_MAX_HEAD (256) #define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8) #define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE \ @@ -1060,13 +1061,15 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv); struct mlx5e_xsk_param; struct mlx5e_rq_param; -int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, - struct mlx5e_xsk_param *xsk, int node, u16 q_counter, +struct mlx5e_rq_opt_param; +int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param, + struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter, struct mlx5e_rq *rq); #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time); void mlx5e_close_rq(struct mlx5e_rq *rq); -int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter); +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param, + u16 q_counter); void mlx5e_destroy_rq(struct mlx5e_rq *rq); bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 8e99d07586fa..26bb31c56e45 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -10,6 +10,7 @@ #include #include #include +#include #define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18 #define MLX5_REP_MPWRQ_MAX_LOG_WQE_SZ 17 @@ -21,10 +22,20 @@ static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev) return min_page_shift ? : 12; } -u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) +u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, + struct mlx5e_rq_opt_param *rqo) { - u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT; + struct netdev_queue_config *qcfg = rqo ? rqo->qcfg : NULL; + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev); + u8 req_page_shift; + + if (xsk) + req_page_shift = order_base_2(xsk->chunk_size); + else if (qcfg && qcfg->rx_page_size) + req_page_shift = order_base_2(qcfg->rx_page_size); + else + req_page_shift = PAGE_SHIFT; /* Regular RQ uses order-0 pages, the NIC must be able to map them. */ if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift)) @@ -34,7 +45,8 @@ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xs } enum mlx5e_mpwrq_umr_mode -mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) +mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, + struct mlx5e_rq_opt_param *rqo) { /* Different memory management schemes use different mechanisms to map * user-mode memory. The stricter guarantees we have, the faster @@ -45,7 +57,8 @@ mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) * 3. KLM - indirect mapping to another MKey to arbitrary addresses, and * mappings can have different sizes. */ - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); bool unaligned = xsk ? xsk->unaligned : false; bool oversized = false; @@ -225,12 +238,12 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, } u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { u16 headroom; - if (xsk) - return xsk->headroom; + if (mlx5e_rqo_xsk_param(rqo)) + return rqo->xsk->headroom; headroom = NET_IP_ALIGN; if (params->xdp_prog) @@ -263,19 +276,23 @@ static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool no_head_ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, bool mpwqe) { + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); bool no_head_tail_room; u32 sz; /* XSK frames are mapped as individual pages, because frames may come in * an arbitrary order from random locations in the UMEM. */ - if (xsk) - return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE; + if (xsk) { + return mpwqe ? + BIT(mlx5e_mpwrq_page_shift(mdev, rqo)) : PAGE_SIZE; + } - no_head_tail_room = params->xdp_prog && mpwqe && !mlx5e_rx_is_linear_skb(mdev, params, xsk); + no_head_tail_room = params->xdp_prog && mpwqe && + !mlx5e_rx_is_linear_skb(mdev, params, rqo); /* When no_head_tail_room is set, headroom and tailroom are excluded from skb calculations. * no_head_tail_room should be set in the case of XDP with Striding RQ @@ -291,11 +308,12 @@ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev, static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true); - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u32 linear_stride_sz = + mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, true); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) - order_base_2(linear_stride_sz); @@ -303,8 +321,10 @@ static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); + if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) return false; @@ -315,7 +335,7 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, * Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data * must fit into a CPU page. */ - if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE) + if (mlx5e_rx_get_linear_sz_skb(params, !!xsk) > PAGE_SIZE) return false; /* XSK frames must be big enough to hold the packet data. */ @@ -349,12 +369,14 @@ static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); - u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u8 log_wqe_num_of_strides = + mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo); + u8 log_wqe_stride_size = + mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); return mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, log_wqe_num_of_strides, @@ -363,18 +385,20 @@ bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev, bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u32 linear_stride_sz = + mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, true); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); u8 log_num_strides; u8 log_stride_sz; u8 log_wqe_sz; - if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) + if (!mlx5e_rx_is_linear_skb(mdev, params, rqo)) return false; - log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); + log_stride_sz = order_base_2(linear_stride_sz); log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode); if (log_wqe_sz < log_stride_sz) @@ -389,13 +413,13 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); u8 log_pkts_per_wqe, page_shift, max_log_rq_size; - log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk); - page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, rqo); + page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode); /* Numbers are unsigned, don't subtract to avoid underflow. */ @@ -423,10 +447,11 @@ static u8 mlx5e_shampo_get_log_pkt_per_rsrv(struct mlx5e_params *params) u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) - return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); + if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo)) + return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, + rqo, true)); /* XDP in mlx5e doesn't support multiple packets per page. */ if (params->xdp_prog) @@ -437,17 +462,18 @@ u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); u8 log_wqe_size, log_stride_size; log_wqe_size = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode); - log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); + log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo); WARN(log_wqe_size < log_stride_size, "Log WQE size %u < log stride size %u (page shift %u, umr mode %d, xsk on? %d)\n", - log_wqe_size, log_stride_size, page_shift, umr_mode, !!xsk); + log_wqe_size, log_stride_size, page_shift, umr_mode, + rqo && rqo->xsk); return log_wqe_size - log_stride_size; } @@ -459,14 +485,14 @@ u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz) u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, xsk); + u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, rqo); if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) return linear_headroom; - if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) + if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo)) return linear_headroom; if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) @@ -475,6 +501,18 @@ u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, return 0; } +u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev) +{ + if (mlx5_core_is_ecpf(mdev)) + return PAGE_SIZE; + + /* Two MTTs are needed to form an octword. Driver is using a + * single page per MTT for simplicity. Hence the limit of having + * at least 2 pages per WQE. + */ + return BIT(MLX5_MPWRQ_MAX_LOG_WQE_SZ - 1); +} + u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params) { bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE); @@ -535,10 +573,11 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params } int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); u16 max_mtu_pkts; if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode)) { @@ -547,7 +586,7 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa return -EOPNOTSUPP; } - if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) { + if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo)) { mlx5_core_err(mdev, "Striding RQ linear mode for XSK can't be activated with current params\n"); return -EINVAL; } @@ -559,7 +598,8 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, xsk->unaligned)); if (params->log_rq_mtu_frames > max_mtu_pkts) { mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n", - 1 << params->log_rq_mtu_frames, xsk->chunk_size); + 1 << params->log_rq_mtu_frames, + xsk->chunk_size); return -EINVAL; } @@ -672,7 +712,7 @@ static void mlx5e_rx_compute_wqe_bulk_params(struct mlx5e_params *params, static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5e_rq_frags_info *info, u32 *xdp_frag_size) { @@ -684,10 +724,11 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, int max_mtu; int i; - if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) { + if (mlx5e_rx_is_linear_skb(mdev, params, rqo)) { int frag_stride; - frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, false); + frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, + false); info->arr[0].frag_size = byte_count; info->arr[0].frag_stride = frag_stride; @@ -703,7 +744,7 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, goto out; } - headroom = mlx5e_get_linear_rq_headroom(params, xsk); + headroom = mlx5e_get_linear_rq_headroom(params, rqo); first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom); max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size_max, frag_size_max, @@ -819,12 +860,13 @@ static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev, static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); - u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); + u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo); + u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, + rqo)); int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(params)); - int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); + int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo)); int wqe_size = BIT(log_stride_sz) * num_strides; int rsrv_size = MLX5E_SHAMPO_WQ_RESRV_SIZE; @@ -836,7 +878,7 @@ static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev, static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5e_cq_param *param) { bool hw_stridx = false; @@ -847,10 +889,13 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev, case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index); if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) - log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk); + log_cq_size = + mlx5e_shampo_get_log_cq_size(mdev, params, rqo); else - log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) + - mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); + log_cq_size = + mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo) + + mlx5e_mpwqe_get_log_num_strides(mdev, params, + rqo); break; default: /* MLX5_WQ_TYPE_CYCLIC */ log_cq_size = params->log_rq_mtu_frames; @@ -880,69 +925,90 @@ static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *param MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN; } +static int mlx5e_mpwqe_build_rq_param(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_rq_opt_param *rqo, + struct mlx5e_rq_param *rq_param) +{ + u8 log_rq_sz = mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); + u8 log_wqe_num_of_strides, log_wqe_stride_size; + enum mlx5e_mpwrq_umr_mode umr_mode; + void *rqc = rq_param->rqc; + u32 lro_timeout; + void *wq; + + log_wqe_num_of_strides = + mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo); + log_wqe_stride_size = + mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo); + umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + + wq = MLX5_ADDR_OF(rqc, rqc, wq); + if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, + log_wqe_num_of_strides, + page_shift, umr_mode)) { + mlx5_core_err(mdev, + "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n", + log_wqe_stride_size, log_wqe_num_of_strides, + umr_mode); + return -EINVAL; + } + + MLX5_SET(wq, wq, log_wqe_num_of_strides, + log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE); + MLX5_SET(wq, wq, log_wqe_stride_size, + log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE); + MLX5_SET(wq, wq, log_wq_sz, log_rq_sz); + if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) + return 0; + + MLX5_SET(wq, wq, shampo_enable, true); + MLX5_SET(wq, wq, log_reservation_size, + MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE - + MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT); + MLX5_SET(wq, wq, log_max_num_of_packets_per_reservation, + mlx5e_shampo_get_log_pkt_per_rsrv(params)); + MLX5_SET(wq, wq, log_headers_entry_size, + MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE - + MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT); + lro_timeout = mlx5e_choose_lro_timeout(mdev, + MLX5E_DEFAULT_SHAMPO_TIMEOUT); + MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout); + MLX5_SET(rqc, rqc, shampo_match_criteria_type, + MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED); + MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity, + MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE); + + return 0; +} + int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, - struct mlx5e_rq_param *param) + struct mlx5e_rq_opt_param *rqo, + struct mlx5e_rq_param *rq_param) { - void *rqc = param->rqc; - void *wq = MLX5_ADDR_OF(rqc, rqc, wq); - u32 lro_timeout; + void *rqc = rq_param->rqc; int ndsegs = 1; + void *wq; int err; + wq = MLX5_ADDR_OF(rqc, rqc, wq); + switch (params->rq_wq_type) { - case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: { - u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); - u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); - - if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, - log_wqe_num_of_strides, - page_shift, umr_mode)) { - mlx5_core_err(mdev, - "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n", - log_wqe_stride_size, log_wqe_num_of_strides, - umr_mode); - return -EINVAL; - } - - MLX5_SET(wq, wq, log_wqe_num_of_strides, - log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE); - MLX5_SET(wq, wq, log_wqe_stride_size, - log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE); - MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); - if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) - break; - - MLX5_SET(wq, wq, shampo_enable, true); - MLX5_SET(wq, wq, log_reservation_size, - MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE - - MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT); - MLX5_SET(wq, wq, - log_max_num_of_packets_per_reservation, - mlx5e_shampo_get_log_pkt_per_rsrv(params)); - MLX5_SET(wq, wq, log_headers_entry_size, - MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE - - MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT); - lro_timeout = - mlx5e_choose_lro_timeout(mdev, - MLX5E_DEFAULT_SHAMPO_TIMEOUT); - MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout); - MLX5_SET(rqc, rqc, shampo_match_criteria_type, - MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED); - MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity, - MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE); - break; - } - default: /* MLX5_WQ_TYPE_CYCLIC */ - MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames); - err = mlx5e_build_rq_frags_info(mdev, params, xsk, ¶m->frags_info, - ¶m->xdp_frag_size); + case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + err = mlx5e_mpwqe_build_rq_param(mdev, params, rqo, rq_param); if (err) return err; - ndsegs = param->frags_info.num_frags; + break; + default: /* MLX5_WQ_TYPE_CYCLIC */ + MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames); + err = mlx5e_build_rq_frags_info(mdev, params, rqo, + &rq_param->frags_info, + &rq_param->xdp_frag_size); + if (err) + return err; + ndsegs = rq_param->frags_info.num_frags; } MLX5_SET(wq, wq, wq_type, params->rq_wq_type); @@ -953,23 +1019,23 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); - param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); - mlx5e_build_rx_cq_param(mdev, params, xsk, ¶m->cqp); + rq_param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); + mlx5e_build_rx_cq_param(mdev, params, rqo, &rq_param->cqp); return 0; } void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, - struct mlx5e_rq_param *param) + struct mlx5e_rq_param *rq_param) { - void *rqc = param->rqc; + void *rqc = rq_param->rqc; void *wq = MLX5_ADDR_OF(rqc, rqc, wq); MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); MLX5_SET(wq, wq, log_wq_stride, mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); - param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); + rq_param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); } void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev, @@ -1084,29 +1150,30 @@ u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); u8 umr_wqebbs; umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); - return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); + return umr_wqebbs * + (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo)); } -static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, - struct mlx5e_params *params, - struct mlx5e_rq_param *rqp) +static u32 mlx5e_max_xsk_wqebbs(struct mlx5_core_dev *mdev, + struct mlx5e_params *params) { - u32 wqebbs, total_pages, useful_space; + struct mlx5e_rq_opt_param rqo = {0}; + struct mlx5e_xsk_param xsk = {0}; + u32 max_xsk_wqebbs = 0; + u8 frame_shift; - /* MLX5_WQ_TYPE_CYCLIC */ - if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) - return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; + if (!params->xdp_prog) + return 0; - /* UMR WQEs for the regular RQ. */ - wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL); + rqo.xsk = &xsk; /* If XDP program is attached, XSK may be turned on at any time without * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of @@ -1118,41 +1185,54 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, * from capabilities. Hence, we have to try all valid values of XSK * frame size (and page_shift) to find the maximum. */ - if (params->xdp_prog) { - u32 max_xsk_wqebbs = 0; - u8 frame_shift; + for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT; + frame_shift <= PAGE_SHIFT; frame_shift++) { + u32 total_wqebbs; - for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT; - frame_shift <= PAGE_SHIFT; frame_shift++) { - /* The headroom doesn't affect the calculation. */ - struct mlx5e_xsk_param xsk = { - .chunk_size = 1 << frame_shift, - .unaligned = false, - }; + /* The headroom doesn't affect the calculations below. */ - /* XSK aligned mode. */ - max_xsk_wqebbs = max(max_xsk_wqebbs, - mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); + /* XSK aligned mode. */ + xsk.chunk_size = 1 << frame_shift; + xsk.unaligned = false; + total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo); + max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs); - /* XSK unaligned mode, frame size is a power of two. */ - xsk.unaligned = true; - max_xsk_wqebbs = max(max_xsk_wqebbs, - mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); + /* XSK unaligned mode, frame size is a power of two. */ + xsk.unaligned = true; + total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo); + max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs); - /* XSK unaligned mode, frame size is not equal to stride size. */ - xsk.chunk_size -= 1; - max_xsk_wqebbs = max(max_xsk_wqebbs, - mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); + /* XSK unaligned mode, frame size is not equal to stride + * size. + */ + xsk.chunk_size -= 1; + total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo); + max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs); - /* XSK unaligned mode, frame size is a triple power of two. */ - xsk.chunk_size = (1 << frame_shift) / 4 * 3; - max_xsk_wqebbs = max(max_xsk_wqebbs, - mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); - } - - wqebbs += max_xsk_wqebbs; + /* XSK unaligned mode, frame size is a triple power of two. */ + xsk.chunk_size = (1 << frame_shift) / 4 * 3; + total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo); + max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs); } + return max_xsk_wqebbs; +} + +static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_rq_param *rq_param) +{ + u32 wqebbs, total_pages, useful_space; + + /* MLX5_WQ_TYPE_CYCLIC */ + if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) + return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; + + /* UMR WQEs for the regular RQ. */ + wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL); + + wqebbs += mlx5e_max_xsk_wqebbs(mdev, params); + /* UMR WQEs don't cross the page boundary, they are padded with NOPs. * This padding is always smaller than the max WQE size. That gives us * at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes @@ -1222,12 +1302,15 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, + struct netdev_queue_config *qcfg, struct mlx5e_channel_param *cparam) { u8 icosq_log_wq_sz, async_icosq_log_wq_sz; int err; - err = mlx5e_build_rq_param(mdev, params, NULL, &cparam->rq); + cparam->rq_opt.qcfg = qcfg; + + err = mlx5e_build_rq_param(mdev, params, &cparam->rq_opt, &cparam->rq); if (err) return err; @@ -1241,3 +1324,13 @@ int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, return 0; } + +void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk, + struct mlx5e_channel_param *cparam) +{ + cparam->rq_opt.xsk = xsk; + mlx5e_build_rq_param(mdev, params, &cparam->rq_opt, &cparam->rq); + mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 00617c65fe3c..9b1a2aed17c3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -12,6 +12,11 @@ struct mlx5e_xsk_param { bool unaligned; }; +struct mlx5e_rq_opt_param { + struct mlx5e_xsk_param *xsk; + struct netdev_queue_config *qcfg; +}; + struct mlx5e_cq_param { u32 cqc[MLX5_ST_SZ_DW(cqc)]; struct mlx5_wq_param wq; @@ -38,6 +43,7 @@ struct mlx5e_sq_param { struct mlx5e_channel_param { struct mlx5e_rq_param rq; + struct mlx5e_rq_opt_param rq_opt; struct mlx5e_sq_param txq_sq; struct mlx5e_sq_param xdp_sq; struct mlx5e_sq_param icosq; @@ -56,9 +62,11 @@ struct mlx5e_create_sq_param { /* Striding RQ dynamic parameters */ -u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); +u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, + struct mlx5e_rq_opt_param *rqo); enum mlx5e_mpwrq_umr_mode -mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); +mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, + struct mlx5e_rq_opt_param *rqo); u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode); u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, enum mlx5e_mpwrq_umr_mode umr_mode); @@ -80,22 +88,22 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, bool slow_pci_heuristic(struct mlx5_core_dev *mdev); int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params); int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params); void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); u32 mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_rq_param *rq_param); @@ -105,21 +113,22 @@ u32 mlx5e_shampo_hd_per_wq(struct mlx5_core_dev *mdev, u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout); u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz); u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); +u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev); /* Build queue parameters */ void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c); int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5e_rq_param *param); void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_rq_param *param); @@ -136,13 +145,19 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, struct mlx5e_sq_param *param); int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, + struct netdev_queue_config *qcfg, struct mlx5e_channel_param *cparam); +void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk, + struct mlx5e_channel_param *cparam); + u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params); int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev, struct mlx5e_params *params) @@ -158,4 +173,10 @@ static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev, "enhanced" : "basic"); }; +static inline struct mlx5e_xsk_param * +mlx5e_rqo_xsk_param(struct mlx5e_rq_opt_param *rqo) +{ + return rqo ? rqo->xsk : NULL; +} + #endif /* __MLX5_EN_PARAMS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 74660e7fe674..13add74d1b97 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -660,13 +660,13 @@ static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev, struct net_device *netdev, struct mlx5e_ptp_params *ptp_params) { - struct mlx5e_rq_param *rq_params = &ptp_params->rq_param; + struct mlx5e_rq_param *rq_param = &ptp_params->rq_param; struct mlx5e_params *params = &ptp_params->params; params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC; mlx5e_init_rq_type_params(mdev, params); params->sw_mtu = netdev->max_mtu; - mlx5e_build_rq_param(mdev, params, NULL, rq_params); + mlx5e_build_rq_param(mdev, params, NULL, rq_param); } static void mlx5e_ptp_build_params(struct mlx5e_ptp *c, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index 80f9fc10877a..04e1b5fa4825 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -37,9 +37,10 @@ #include #include -int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) +int mlx5e_xdp_max_mtu(struct mlx5e_params *params, + struct mlx5e_rq_opt_param *rqo) { - int hr = mlx5e_get_linear_rq_headroom(params, xsk); + int hr = mlx5e_get_linear_rq_headroom(params, rqo); /* Let S := SKB_DATA_ALIGN(sizeof(struct skb_shared_info)). * The condition checked in mlx5e_rx_is_linear_skb is: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index 46ab0a9e8cdd..3c54f8962664 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -96,7 +96,8 @@ union mlx5e_xdp_info { }; struct mlx5e_xsk_param; -int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); +int mlx5e_xdp_max_mtu(struct mlx5e_params *params, + struct mlx5e_rq_opt_param *rqo); bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx); void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c index 5c5360a25c64..565e5c4ddcce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c @@ -79,6 +79,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, struct xsk_buff_pool *pool, u16 ix) { struct mlx5e_params *params = &priv->channels.params; + struct mlx5e_channel_param *cparam; + enum mlx5e_mpwrq_umr_mode umr_mode; struct mlx5e_xsk_param xsk; struct mlx5e_channel *c; int err; @@ -89,18 +91,24 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, if (unlikely(!mlx5e_xsk_is_pool_sane(pool))) return -EINVAL; + cparam = kvzalloc_obj(*cparam, GFP_KERNEL); + if (!cparam) + return -ENOMEM; + err = mlx5e_xsk_map_pool(mlx5_sd_ch_ix_get_dev(priv->mdev, ix), pool); if (unlikely(err)) - return err; + goto err_free_cparam; err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix); if (unlikely(err)) goto err_unmap_pool; mlx5e_build_xsk_param(pool, &xsk); + mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam); + umr_mode = mlx5e_mpwrq_umr_mode(priv->mdev, &cparam->rq_opt); if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && - mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) { + umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) { const char *recommendation = is_power_of_2(xsk.chunk_size) ? "Upgrade firmware" : "Disable striding RQ"; @@ -122,7 +130,7 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, c = priv->channels.c[ix]; - err = mlx5e_open_xsk(priv, params, &xsk, pool, c); + err = mlx5e_open_xsk(priv, params, cparam, pool, c); if (unlikely(err)) goto err_remove_pool; @@ -138,6 +146,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, mlx5e_deactivate_rq(&c->rq); mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY); + kvfree(cparam); + return 0; err_remove_pool: @@ -146,17 +156,22 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, err_unmap_pool: mlx5e_xsk_unmap_pool(priv, pool); +err_free_cparam: + kvfree(cparam); + return err; validate_closed: /* Check the configuration in advance, rather than fail at a later stage * (in mlx5e_xdp_set or on open) and end up with no channels. */ - if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) { + if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev)) { err = -EINVAL; goto err_remove_pool; } + kvfree(cparam); + return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index 5981c71cae2d..11500fd213a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -9,9 +9,9 @@ static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) { + if (!mlx5e_rx_is_linear_skb(mdev, params, rqo)) { mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n"); return -EINVAL; } @@ -25,9 +25,14 @@ static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev, #define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE) bool mlx5e_validate_xsk_param(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5_core_dev *mdev) { + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); + + if (WARN_ON(!xsk)) + return false; + /* AF_XDP doesn't support frames larger than PAGE_SIZE, * and xsk->chunk_size is limited to 65535 bytes. */ @@ -42,21 +47,12 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params, */ switch (params->rq_wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: - return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk); + return !mlx5e_mpwrq_validate_xsk(mdev, params, rqo); default: /* MLX5_WQ_TYPE_CYCLIC */ - return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk); + return !mlx5e_legacy_rq_validate_xsk(mdev, params, rqo); } } -static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev, - struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, - struct mlx5e_channel_param *cparam) -{ - mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq); - mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq); -} - static int mlx5e_init_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params, struct xsk_buff_pool *pool, @@ -90,19 +86,23 @@ static int mlx5e_init_xsk_rq(struct mlx5e_channel *c, return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id); } -static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params, - struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool, - struct mlx5e_xsk_param *xsk) +static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, + struct mlx5e_params *params, + struct mlx5e_channel_param *cparam, + struct xsk_buff_pool *pool) { + struct mlx5e_rq_param *rq_param = &cparam->rq; + struct mlx5e_rq_opt_param *rqo = &cparam->rq_opt; u16 q_counter = c->priv->q_counter[c->sd_ix]; struct mlx5e_rq *xskrq = &c->xskrq; int err; - err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq); + err = mlx5e_init_xsk_rq(c, params, pool, rqo->xsk, xskrq); if (err) return err; - err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq); + err = mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu), + q_counter, xskrq); if (err) return err; @@ -111,30 +111,24 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param } int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool, + struct mlx5e_channel_param *cparam, + struct xsk_buff_pool *pool, struct mlx5e_channel *c) { - struct mlx5e_channel_param *cparam; struct mlx5e_create_cq_param ccp; int err; mlx5e_build_create_cq_param(&ccp, c); - if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev)) + if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev)) return -EINVAL; - cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL); - if (!cparam) - return -ENOMEM; - - mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam); - err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, &c->xskrq.cq); if (unlikely(err)) - goto err_free_cparam; + return err; - err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk); + err = mlx5e_open_xsk_rq(c, params, cparam, pool); if (unlikely(err)) goto err_close_rx_cq; @@ -153,8 +147,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (unlikely(err)) goto err_close_tx_cq; - kvfree(cparam); - set_bit(MLX5E_CHANNEL_STATE_XSK, c->state); return 0; @@ -168,9 +160,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, err_close_rx_cq: mlx5e_close_cq(&c->xskrq.cq); -err_free_cparam: - kvfree(cparam); - return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h index 50e111b85efd..664ec78192c3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h @@ -9,10 +9,12 @@ struct mlx5e_xsk_param; bool mlx5e_validate_xsk_param(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5_core_dev *mdev); +struct mlx5e_channel_param; int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool, + struct mlx5e_channel_param *cparam, + struct xsk_buff_pool *pool, struct mlx5e_channel *c); void mlx5e_close_xsk(struct mlx5e_channel *c); void mlx5e_activate_xsk(struct mlx5e_channel *c); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 7eb691c2a1bd..67dc38981101 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -636,14 +636,18 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) { - rq->wqe_overflow.page = alloc_page(GFP_KERNEL); + /* xsk can have page_shift < PAGE_SHIFT */ + u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0); + u32 page_size = BIT(PAGE_SHIFT + page_order); + + rq->wqe_overflow.page = alloc_pages(GFP_KERNEL, page_order); if (!rq->wqe_overflow.page) return -ENOMEM; rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0, - PAGE_SIZE, rq->buff.map_dir); + page_size, rq->buff.map_dir); if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) { - __free_page(rq->wqe_overflow.page); + __free_pages(rq->wqe_overflow.page, page_order); return -ENOMEM; } return 0; @@ -651,9 +655,12 @@ static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq) { - dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE, - rq->buff.map_dir); - __free_page(rq->wqe_overflow.page); + u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0); + u32 page_size = BIT(PAGE_SHIFT + page_order); + + dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, page_size, + rq->buff.map_dir); + __free_pages(rq->wqe_overflow.page, page_order); } static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, @@ -780,7 +787,7 @@ static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev, static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_rq_param *rqp, + struct mlx5e_rq_param *rq_param, struct mlx5e_rq *rq, int node) { @@ -791,7 +798,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) return 0; - hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp); + hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rq_param); hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE); nentries = hd_buf_size / PAGE_SIZE; if (!nentries) { @@ -851,19 +858,19 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) } static int mlx5e_alloc_rq(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, - struct mlx5e_rq_param *rqp, + struct mlx5e_rq_param *rq_param, + struct mlx5e_rq_opt_param *rqo, int node, struct mlx5e_rq *rq) { + void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); struct mlx5_core_dev *mdev = rq->mdev; - void *rqc = rqp->rqc; - void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); + u32 pool_order = 0; u32 pool_size; int wq_sz; int err; int i; - rqp->wq.db_numa_node = node; + rq_param->wq.db_numa_node = node; INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); INIT_WORK(&rq->rx_timeout_work, mlx5e_rq_timeout_work); @@ -872,28 +879,28 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog); rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; - rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); + rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, rqo); pool_size = 1 << params->log_rq_mtu_frames; rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: - err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, - &rq->wq_ctrl); + err = mlx5_wq_ll_create(mdev, &rq_param->wq, rqc_wq, + &rq->mpwqe.wq, &rq->wq_ctrl); if (err) goto err_rq_xdp_prog; - err = mlx5e_alloc_mpwqe_rq_drop_page(rq); - if (err) - goto err_rq_wq_destroy; - rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); - rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); - rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); + err = mlx5e_alloc_mpwqe_rq_drop_page(rq); + if (err) + goto err_rq_wq_destroy; + + rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); rq->mpwqe.pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, rq->mpwqe.umr_mode); @@ -905,14 +912,18 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, rq->mpwqe.umr_mode); pool_size = rq->mpwqe.pages_per_wqe << - mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); + mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo); + pool_order = rq->mpwqe.page_shift - PAGE_SHIFT; - if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog) + if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo) && + params->xdp_prog) pool_size *= 2; /* additional page per packet for the linear part */ - rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); + rq->mpwqe.log_stride_sz = + mlx5e_mpwqe_get_log_stride_size(mdev, params, + rqo); rq->mpwqe.num_strides = - BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); + BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo)); rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz); rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz); @@ -925,14 +936,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, if (err) goto err_rq_mkey; - err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, node); + err = mlx5_rq_shampo_alloc(mdev, params, rq_param, rq, node); if (err) goto err_free_mpwqe_info; break; default: /* MLX5_WQ_TYPE_CYCLIC */ - err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, - &rq->wq_ctrl); + err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, + &rq->wqe.wq, &rq->wq_ctrl); if (err) goto err_rq_xdp_prog; @@ -940,7 +951,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); - rq->wqe.info = rqp->frags_info; + rq->wqe.info = rq_param->frags_info; rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; err = mlx5e_init_wqe_alloc_info(rq, node); @@ -948,7 +959,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, goto err_rq_wq_destroy; } - if (xsk) { + if (mlx5e_rqo_xsk_param(rqo)) { err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); if (err) @@ -958,7 +969,13 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, /* Create a page_pool and register it with rxq */ struct page_pool_params pp_params = { 0 }; - pp_params.order = 0; + if (WARN_ON(BIT(PAGE_SHIFT + pool_order) / 64 > + MLX5E_PAGECNT_BIAS_MAX)) { + err = -E2BIG; + goto err_free_by_rq_type; + } + + pp_params.order = pool_order; pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; pp_params.pool_size = pool_size; pp_params.nid = node; @@ -966,7 +983,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, pp_params.napi = rq->cq.napi; pp_params.netdev = rq->netdev; pp_params.dma_dir = rq->buff.map_dir; - pp_params.max_len = PAGE_SIZE; + pp_params.max_len = BIT(PAGE_SHIFT + pool_order); pp_params.queue_idx = rq->ix; /* Shampo header data split allow for unreadable netmem */ @@ -1085,7 +1102,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) xdp_rxq_info_unreg(&rq->xdp_rxq); } -int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter) +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param, + u16 q_counter) { struct mlx5_core_dev *mdev = rq->mdev; u8 ts_format; @@ -1107,7 +1125,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_cou rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); wq = MLX5_ADDR_OF(rqc, rqc, wq); - memcpy(rqc, param->rqc, sizeof(param->rqc)); + memcpy(rqc, rq_param->rqc, sizeof(rq_param->rqc)); MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); @@ -1323,8 +1341,8 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq) } -int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, - struct mlx5e_xsk_param *xsk, int node, u16 q_counter, +int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param, + struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter, struct mlx5e_rq *rq) { struct mlx5_core_dev *mdev = rq->mdev; @@ -1333,11 +1351,11 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state); - err = mlx5e_alloc_rq(params, xsk, param, node, rq); + err = mlx5e_alloc_rq(params, rq_param, rqo, node, rq); if (err) return err; - err = mlx5e_create_rq(rq, param, q_counter); + err = mlx5e_create_rq(rq, rq_param, q_counter); if (err) goto err_free_rq; @@ -2506,17 +2524,20 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) return err; } -static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, - struct mlx5e_rq_param *rq_params) +static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, + struct mlx5e_params *params, + struct mlx5e_rq_param *rq_param, + struct mlx5e_rq_opt_param *rqo) { u16 q_counter = c->priv->q_counter[c->sd_ix]; int err; - err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq); + err = mlx5e_init_rxq_rq(c, params, rq_param->xdp_frag_size, &c->rq); if (err) return err; - return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq); + return mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu), + q_counter, &c->rq); } static struct mlx5e_icosq * @@ -2619,7 +2640,7 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, if (err) goto err_close_icosq; - err = mlx5e_open_rxq_rq(c, params, &cparam->rq); + err = mlx5e_open_rxq_rq(c, params, &cparam->rq, &cparam->rq_opt); if (err) goto err_close_sqs; @@ -2764,6 +2785,7 @@ static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c) static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, struct mlx5e_params *params, + struct netdev_queue_config *qcfg, struct xsk_buff_pool *xsk_pool, struct mlx5e_channel **cp) { @@ -2797,7 +2819,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, goto err_free; } - err = mlx5e_build_channel_param(mdev, params, cparam); + err = mlx5e_build_channel_param(mdev, params, qcfg, cparam); if (err) goto err_free; @@ -2828,7 +2850,8 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, if (xsk_pool) { mlx5e_build_xsk_param(xsk_pool, &xsk); - err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c); + mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam); + err = mlx5e_open_xsk(priv, params, cparam, xsk_pool, c); if (unlikely(err)) goto err_close_queues; } @@ -2921,7 +2944,8 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, if (chs->params.xdp_prog) xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i); - err = mlx5e_open_channel(priv, i, &chs->params, xsk_pool, &chs->c[i]); + err = mlx5e_open_channel(priv, i, &chs->params, NULL, + xsk_pool, &chs->c[i]); if (err) goto err_close_channels; } @@ -3577,15 +3601,14 @@ static void mlx5e_free_drop_rq(struct mlx5e_rq *rq) static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq, - struct mlx5e_rq_param *param) + struct mlx5e_rq_param *rq_param) { - void *rqc = param->rqc; - void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); + void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); int err; - param->wq.db_numa_node = param->wq.buf_numa_node; + rq_param->wq.db_numa_node = rq_param->wq.buf_numa_node; - err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, + err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, &rq->wqe.wq, &rq->wq_ctrl); if (err) return err; @@ -4586,6 +4609,7 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, for (ix = 0; ix < chs->params.num_channels; ix++) { struct xsk_buff_pool *xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix); + struct mlx5e_rq_opt_param rqo = {0}; struct mlx5e_xsk_param xsk; int max_xdp_mtu; @@ -4593,12 +4617,13 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, continue; mlx5e_build_xsk_param(xsk_pool, &xsk); - max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk); + rqo.xsk = &xsk; + max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &rqo); /* Validate XSK params and XDP MTU in advance */ - if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) || + if (!mlx5e_validate_xsk_param(new_params, &rqo, mdev) || new_params->sw_mtu > max_xdp_mtu) { - u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk); + u32 hr = mlx5e_get_linear_rq_headroom(new_params, &rqo); int max_mtu_frame, max_mtu_page, max_mtu; /* Two criteria must be met: @@ -5549,9 +5574,52 @@ static const struct netdev_stat_ops mlx5e_stat_ops = { struct mlx5_qmgmt_data { struct mlx5e_channel *c; - struct mlx5e_channel_param cparam; }; +static void mlx5e_queue_default_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg) +{ + qcfg->rx_page_size = PAGE_SIZE; +} + +static int mlx5e_queue_validate_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = netdev_priv(dev); + struct mlx5_core_dev *mdev = priv->mdev; + u32 max; + + if (!is_power_of_2(qcfg->rx_page_size)) { + netdev_err(priv->netdev, "rx_page_size not power of 2: %u", + qcfg->rx_page_size); + return -EINVAL; + } + + max = mlx5e_mpwrq_max_page_size(mdev); + if (qcfg->rx_page_size < PAGE_SIZE || qcfg->rx_page_size > max) { + netdev_err(priv->netdev, + "Selected rx_page_size %u not in supported range [%lu, %u]\n", + qcfg->rx_page_size, PAGE_SIZE, max); + return -ERANGE; + } + + return 0; +} + +static bool mlx5e_queue_validate_page_size(struct net_device *dev, + struct netdev_queue_config *qcfg, + int queue_index) +{ + if (qcfg->rx_page_size == PAGE_SIZE) + return true; + + if (!netif_rxq_has_unreadable_mp(dev, queue_index)) + return false; + + return true; +} + static int mlx5e_queue_mem_alloc(struct net_device *dev, struct netdev_queue_config *qcfg, void *newq, int queue_index) @@ -5560,7 +5628,6 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev, struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_channels *chs = &priv->channels; struct mlx5e_params params = chs->params; - struct mlx5_core_dev *mdev; int err; mutex_lock(&priv->state_lock); @@ -5584,12 +5651,14 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev, goto unlock; } - mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, queue_index); - err = mlx5e_build_channel_param(mdev, ¶ms, &new->cparam); - if (err) + if (!mlx5e_queue_validate_page_size(dev, qcfg, queue_index)) { + netdev_err(priv->netdev, "High order pages are supported only in Zero-Copy mode\n"); + err = -EINVAL; goto unlock; + } - err = mlx5e_open_channel(priv, queue_index, ¶ms, NULL, &new->c); + err = mlx5e_open_channel(priv, queue_index, ¶ms, qcfg, NULL, + &new->c); unlock: mutex_unlock(&priv->state_lock); return err; @@ -5668,6 +5737,9 @@ static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = { .ndo_queue_start = mlx5e_queue_start, .ndo_queue_stop = mlx5e_queue_stop, .ndo_queue_get_dma_dev = mlx5e_queue_get_dma_dev, + .ndo_default_qcfg = mlx5e_queue_default_qcfg, + .ndo_validate_qcfg = mlx5e_queue_validate_qcfg, + .supported_params = QCFG_RX_PAGE_SIZE, }; static void mlx5e_build_nic_netdev(struct net_device *netdev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index efcfcddab376..8fb57a4f36dd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -272,8 +272,6 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq, return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem); } -#define MLX5E_PAGECNT_BIAS_MAX (PAGE_SIZE / 64) - static int mlx5e_page_alloc_fragmented(struct page_pool *pp, struct mlx5e_frag_page *frag_page) { @@ -1847,11 +1845,14 @@ mlx5e_shampo_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_frag_page *frag_page, u32 data_bcnt, u32 data_offset) { + u32 page_size = BIT(rq->mpwqe.page_shift); + net_prefetchw(skb->data); do { /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ - u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt); + u32 pg_consumed_bytes = min_t(u32, page_size - data_offset, + data_bcnt); unsigned int truesize = pg_consumed_bytes; mlx5e_add_skb_frag(rq, skb, frag_page, data_offset, @@ -1872,6 +1873,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt); struct mlx5e_frag_page *head_page = frag_page; struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf; + u32 page_size = BIT(rq->mpwqe.page_shift); u32 frag_offset = head_offset; u32 byte_cnt = cqe_bcnt; struct skb_shared_info *sinfo; @@ -1926,9 +1928,9 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w linear_hr = skb_headroom(skb); linear_data_len = headlen; linear_frame_sz = MLX5_SKB_FRAG_SZ(skb_end_offset(skb)); - if (unlikely(frag_offset >= PAGE_SIZE)) { + if (unlikely(frag_offset >= page_size)) { frag_page++; - frag_offset -= PAGE_SIZE; + frag_offset -= page_size; } } @@ -1940,7 +1942,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w while (byte_cnt) { /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ pg_consumed_bytes = - min_t(u32, PAGE_SIZE - frag_offset, byte_cnt); + min_t(u32, page_size - frag_offset, byte_cnt); if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) truesize += pg_consumed_bytes; @@ -1978,7 +1980,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w nr_frags_free = old_nr_frags - sinfo->nr_frags; if (unlikely(nr_frags_free)) { frag_page -= nr_frags_free; - truesize -= (nr_frags_free - 1) * PAGE_SIZE + + truesize -= (nr_frags_free - 1) * page_size + ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); } @@ -2166,15 +2168,16 @@ mlx5e_shampo_flush_skb(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match) rq->hw_gro_data->skb = NULL; } -static bool -mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt) +static bool mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, + u16 data_bcnt, + u32 page_size) { int nr_frags = skb_shinfo(skb)->nr_frags; - if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE) + if (page_size >= GRO_LEGACY_MAX_SIZE) return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE; else - return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; + return page_size * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; } static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) @@ -2183,18 +2186,19 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq u16 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe); u32 wqe_offset = be32_to_cpu(cqe->shampo.data_offset); u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); - u32 data_offset = wqe_offset & (PAGE_SIZE - 1); u32 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe); u16 wqe_id = be16_to_cpu(cqe->wqe_id); - u32 page_idx = wqe_offset >> PAGE_SHIFT; u16 head_size = cqe->shampo.header_size; struct sk_buff **skb = &rq->hw_gro_data->skb; bool flush = cqe->shampo.flush; bool match = cqe->shampo.match; + u32 page_size = BIT(rq->mpwqe.page_shift); struct mlx5e_rq_stats *stats = rq->stats; struct mlx5e_rx_wqe_ll *wqe; struct mlx5e_mpw_info *wi; struct mlx5_wq_ll *wq; + u32 data_offset; + u32 page_idx; wi = mlx5e_get_mpw_info(rq, wqe_id); wi->consumed_strides += cstrides; @@ -2210,7 +2214,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq goto mpwrq_cqe_out; } - if (*skb && (!match || !(mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt)))) { + data_offset = wqe_offset & (page_size - 1); + page_idx = wqe_offset >> rq->mpwqe.page_shift; + if (*skb && + !(match && mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt, + page_size))) { match = false; mlx5e_shampo_flush_skb(rq, cqe, match); }