diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index d507289096c2..45bda7b226e9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -1028,7 +1028,7 @@ struct mlx5e_profile { void (*cleanup_rx)(struct mlx5e_priv *priv); int (*init_tx)(struct mlx5e_priv *priv); void (*cleanup_tx)(struct mlx5e_priv *priv); - void (*enable)(struct mlx5e_priv *priv); + int (*enable)(struct mlx5e_priv *priv); void (*disable)(struct mlx5e_priv *priv); int (*update_rx)(struct mlx5e_priv *priv); void (*update_stats)(struct mlx5e_priv *priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index 091b80a67189..4973fb473ff0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -88,13 +88,18 @@ enum { #ifdef CONFIG_MLX5_EN_ARFS MLX5E_ARFS_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, #endif -#if defined(CONFIG_MLX5_EN_IPSEC) || defined(CONFIG_MLX5_EN_PSP) +#if defined(CONFIG_MLX5_EN_IPSEC) MLX5E_ACCEL_FS_ESP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_ACCEL_FS_POL_MISS_FT_LEVEL, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, #endif +#if defined(CONFIG_MLX5_EN_PSP) + MLX5E_ACCEL_FS_PSP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1, + MLX5E_ACCEL_FS_PSP_ERR_FT_LEVEL, + MLX5E_ACCEL_FS_PSP_RX_FT_LEVEL, +#endif }; struct mlx5e_flow_steering; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index b526b3898c22..3f212e46fc2f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -220,18 +220,7 @@ static inline void mlx5e_accel_tx_finish(struct mlx5e_txqsq *sq, static inline int mlx5e_accel_init_rx(struct mlx5e_priv *priv) { - int err; - - err = mlx5_accel_psp_fs_init_rx_tables(priv); - if (err) - goto out; - - err = mlx5e_ktls_init_rx(priv); - if (err) - mlx5_accel_psp_fs_cleanup_rx_tables(priv); - -out: - return err; + return mlx5e_ktls_init_rx(priv); } static inline void mlx5e_accel_cleanup_rx(struct mlx5e_priv *priv) @@ -242,12 +231,6 @@ static inline void mlx5e_accel_cleanup_rx(struct mlx5e_priv *priv) static inline int mlx5e_accel_init_tx(struct mlx5e_priv *priv) { - int err; - - err = mlx5_accel_psp_fs_init_tx_tables(priv); - if (err) - return err; - return mlx5e_ktls_init_tx(priv); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c index d9adb993e64d..73b232379263 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.c @@ -21,51 +21,52 @@ enum accel_psp_syndrome { PSP_BAD_TRAILER, }; -struct mlx5e_psp_tx { +struct mlx5e_psp_tx_table { struct mlx5_flow_namespace *ns; struct mlx5_flow_table *ft; struct mlx5_flow_group *fg; struct mlx5_flow_handle *rule; - struct mutex mutex; /* Protect PSP TX steering */ - u32 refcnt; - struct mlx5_fc *tx_counter; }; -struct mlx5e_psp_rx_err { +struct mlx5e_psp_rx_check_table { struct mlx5_flow_table *ft; + struct mlx5_flow_group *drop_group; struct mlx5_flow_handle *rule; struct mlx5_flow_handle *auth_fail_rule; struct mlx5_flow_handle *err_rule; struct mlx5_flow_handle *bad_rule; - struct mlx5_modify_hdr *copy_modify_hdr; }; -struct mlx5e_accel_fs_psp_prot { +struct mlx5e_psp_rx_decrypt_table { struct mlx5_flow_table *ft; struct mlx5_flow_group *miss_group; struct mlx5_flow_handle *miss_rule; struct mlx5_modify_hdr *rx_modify_hdr; - struct mlx5_flow_destination default_dest; - struct mlx5e_psp_rx_err rx_err; - u32 refcnt; - struct mutex prot_mutex; /* protect ESP4/ESP6 protocol */ - struct mlx5_flow_handle *def_rule; + struct mlx5_flow_handle *rule; }; -struct mlx5e_accel_fs_psp { - struct mlx5e_accel_fs_psp_prot fs_prot[ACCEL_FS_PSP_NUM_TYPES]; - struct mlx5_fc *rx_counter; - struct mlx5_fc *rx_auth_fail_counter; - struct mlx5_fc *rx_err_counter; - struct mlx5_fc *rx_bad_counter; +struct mlx5e_psp_rx_table { + struct mlx5_flow_table *ft; + struct mlx5_flow_group *miss_group; + struct mlx5_flow_handle *miss_rule; + struct mlx5_flow_handle *udp_rules[ACCEL_FS_PSP_NUM_TYPES]; }; struct mlx5e_psp_fs { struct mlx5_core_dev *mdev; - struct mlx5e_psp_tx *tx_fs; - /* Rx manage */ + struct mlx5_fc *tx_counter; + struct mlx5e_psp_tx_table tx; + + /* Rx */ struct mlx5e_flow_steering *fs; - struct mlx5e_accel_fs_psp *rx_fs; + struct mlx5_fc *rx_counter; + struct mlx5_fc *rx_auth_fail_counter; + struct mlx5_fc *rx_err_counter; + struct mlx5_fc *rx_bad_counter; + + struct mlx5e_psp_rx_decrypt_table decrypt[ACCEL_FS_PSP_NUM_TYPES]; + struct mlx5e_psp_rx_check_table check; + struct mlx5e_psp_rx_table rx; }; /* PSP RX flow steering */ @@ -77,46 +78,202 @@ static enum mlx5_traffic_types fs_psp2tt(enum accel_fs_psp_type i) return MLX5_TT_IPV6_UDP; } -static void accel_psp_fs_rx_err_del_rules(struct mlx5e_psp_fs *fs, - struct mlx5e_psp_rx_err *rx_err) +static int accel_psp_fs_create_ft(struct mlx5e_psp_fs *fs, + struct mlx5_flow_table_attr *ft_attr, + struct mlx5_flow_table **ft) { - if (rx_err->bad_rule) { - mlx5_del_flow_rules(rx_err->bad_rule); - rx_err->bad_rule = NULL; + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs->fs, false); + int err = 0; + + *ft = mlx5_create_auto_grouped_flow_table(ns, ft_attr); + if (IS_ERR(*ft)) { + err = PTR_ERR(*ft); + *ft = NULL; } - if (rx_err->err_rule) { - mlx5_del_flow_rules(rx_err->err_rule); - rx_err->err_rule = NULL; - } + return err; +} - if (rx_err->auth_fail_rule) { - mlx5_del_flow_rules(rx_err->auth_fail_rule); - rx_err->auth_fail_rule = NULL; - } - - if (rx_err->rule) { - mlx5_del_flow_rules(rx_err->rule); - rx_err->rule = NULL; - } - - if (rx_err->copy_modify_hdr) { - mlx5_modify_header_dealloc(fs->mdev, rx_err->copy_modify_hdr); - rx_err->copy_modify_hdr = NULL; +static void accel_psp_fs_destroy_ft(struct mlx5_flow_table **table) +{ + if (*table) { + mlx5_destroy_flow_table(*table); + *table = NULL; } } -static void accel_psp_fs_rx_err_destroy_ft(struct mlx5e_psp_fs *fs, - struct mlx5e_psp_rx_err *rx_err) +static void accel_psp_fs_del_flow_rule(struct mlx5_flow_handle **rule) { - accel_psp_fs_rx_err_del_rules(fs, rx_err); - - if (rx_err->ft) { - mlx5_destroy_flow_table(rx_err->ft); - rx_err->ft = NULL; + if (*rule) { + mlx5_del_flow_rules(*rule); + *rule = NULL; } } +static int accel_psp_fs_create_miss_group(struct mlx5_flow_table *ft, + struct mlx5_flow_group **group) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + u32 *in = kvzalloc(inlen, GFP_KERNEL); + int err = 0; + + if (!in) + return -ENOMEM; + + MLX5_SET(create_flow_group_in, in, start_flow_index, ft->max_fte - 1); + MLX5_SET(create_flow_group_in, in, end_flow_index, ft->max_fte - 1); + *group = mlx5_create_flow_group(ft, in); + if (IS_ERR(*group)) { + err = PTR_ERR(*group); + *group = NULL; + } + kvfree(in); + + return err; +} + +static void accel_psp_fs_destroy_flow_group(struct mlx5_flow_group **group) +{ + if (*group) { + mlx5_destroy_flow_group(*group); + *group = NULL; + } +} + +static int accel_psp_fs_create_counter(struct mlx5_core_dev *dev, + struct mlx5_fc **counter) +{ + *counter = mlx5_fc_create(dev, false); + if (IS_ERR(*counter)) { + int err = PTR_ERR(*counter); + + *counter = NULL; + return err; + } + + return 0; +} + +static void accel_psp_fs_destroy_counter(struct mlx5_core_dev *dev, + struct mlx5_fc **counter) +{ + if (*counter) { + mlx5_fc_destroy(dev, *counter); + *counter = NULL; + } +} + +static void accel_psp_fs_rx_ft_destroy(struct mlx5e_psp_rx_table *rx) +{ + int i; + + for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) + accel_psp_fs_del_flow_rule(&rx->udp_rules[i]); + accel_psp_fs_del_flow_rule(&rx->miss_rule); + accel_psp_fs_destroy_flow_group(&rx->miss_group); + accel_psp_fs_destroy_ft(&rx->ft); +} + +static int accel_psp_fs_rx_ft_create(struct mlx5e_psp_fs *fs, + struct mlx5e_psp_rx_table *rx) +{ + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false); + struct mlx5_flow_destination dest[2] = {}; + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_core_dev *mdev = fs->mdev; + MLX5_DECLARE_FLOW_ACT(flow_act); + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int i, err = 0; + + spec = kzalloc_obj(*spec); + if (!spec) + return -ENOMEM; + + ft_attr.max_fte = 1 + ACCEL_FS_PSP_NUM_TYPES; + ft_attr.level = MLX5E_ACCEL_FS_PSP_RX_FT_LEVEL; + ft_attr.prio = MLX5E_NIC_PRIO; + ft_attr.autogroup.num_reserved_entries = 1; + err = accel_psp_fs_create_ft(fs, &ft_attr, &rx->ft); + if (err) { + mlx5_core_err(mdev, "fail to create psp rx ft err=%d\n", err); + goto out_err; + } + + err = accel_psp_fs_create_miss_group(rx->ft, &rx->miss_group); + if (err) { + mlx5_core_err(mdev, "fail to create psp rx miss_group err=%d\n", + err); + goto out_err; + } + + /* Add miss rule */ + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL; + dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest[0].ft = mlx5_get_ttc_flow_table(ttc); + dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[1].counter = fs->rx_counter; + rule = mlx5_add_flow_rules(rx->ft, NULL, &flow_act, dest, 2); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "fail to create psp rx rule, err=%d\n", + err); + goto out_err; + } + rx->miss_rule = rule; + + /* Add UDP v4/v6 rules */ + spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.ip_version); + MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, + ip_protocol); + MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, + IPPROTO_UDP); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + flow_act.flags = 0; + for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { + int version = i == ACCEL_FS_PSP4 ? 4 : 6; + + MLX5_SET(fte_match_param, spec->match_value, + outer_headers.ip_version, version); + dest[0] = mlx5_ttc_get_default_dest(ttc, fs_psp2tt(i)); + dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[1].counter = fs->rx_counter; + rule = mlx5_add_flow_rules(rx->ft, spec, &flow_act, dest, + 2); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, + "fail to create psp rx UDP%d rule err=%d\n", + version, err); + goto out_err; + } + rx->udp_rules[i] = rule; + } + goto out_spec; + +out_err: + accel_psp_fs_rx_ft_destroy(rx); +out_spec: + kvfree(spec); + return err; +} + +static +void accel_psp_fs_rx_check_ft_destroy(struct mlx5e_psp_rx_check_table *check) +{ + accel_psp_fs_del_flow_rule(&check->bad_rule); + accel_psp_fs_del_flow_rule(&check->err_rule); + accel_psp_fs_del_flow_rule(&check->auth_fail_rule); + accel_psp_fs_del_flow_rule(&check->rule); + accel_psp_fs_destroy_flow_group(&check->drop_group); + accel_psp_fs_destroy_ft(&check->ft); +} + static void accel_psp_setup_syndrome_match(struct mlx5_flow_spec *spec, enum accel_psp_syndrome syndrome) { @@ -129,15 +286,35 @@ static void accel_psp_setup_syndrome_match(struct mlx5_flow_spec *spec, MLX5_SET(fte_match_set_misc2, misc_params_2, psp_syndrome, syndrome); } -static int accel_psp_fs_rx_err_add_rule(struct mlx5e_psp_fs *fs, - struct mlx5e_accel_fs_psp_prot *fs_prot, - struct mlx5e_psp_rx_err *rx_err) +static int accel_psp_add_drop_rule(struct mlx5_flow_table *ft, + struct mlx5_flow_spec *spec, + struct mlx5_fc *counter, + struct mlx5_flow_handle **rule) { - u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; - struct mlx5_core_dev *mdev = fs->mdev; - struct mlx5_flow_destination dest[2]; + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_act flow_act = {}; + int err = 0; + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest.counter = counter; + *rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); + if (IS_ERR(*rule)) { + err = PTR_ERR(*rule); + *rule = NULL; + } + return err; +} + +static +int accel_psp_fs_rx_check_ft_create(struct mlx5e_psp_fs *fs, + struct mlx5e_psp_rx_check_table *check) +{ + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_destination dest = {}; + struct mlx5_core_dev *mdev = fs->mdev; struct mlx5_flow_act flow_act = {}; - struct mlx5_modify_hdr *modify_hdr; struct mlx5_flow_handle *fte; struct mlx5_flow_spec *spec; int err = 0; @@ -146,174 +323,97 @@ static int accel_psp_fs_rx_err_add_rule(struct mlx5e_psp_fs *fs, if (!spec) return -ENOMEM; - /* Action to copy 7 bit psp_syndrome to regB[23:29] */ - MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); - MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_PSP_SYNDROME); - MLX5_SET(copy_action_in, action, src_offset, 0); - MLX5_SET(copy_action_in, action, length, 7); - MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); - MLX5_SET(copy_action_in, action, dst_offset, 23); + ft_attr.max_fte = 4; + ft_attr.autogroup.num_reserved_entries = 1; + ft_attr.autogroup.max_num_groups = 2; + ft_attr.level = MLX5E_ACCEL_FS_PSP_ERR_FT_LEVEL; + ft_attr.prio = MLX5E_NIC_PRIO; + err = accel_psp_fs_create_ft(fs, &ft_attr, &check->ft); + if (err) { + mlx5_core_err(fs->mdev, + "fail to create psp rx check ft err=%d\n", err); + goto out_err; + } - modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, - 1, action); - if (IS_ERR(modify_hdr)) { - err = PTR_ERR(modify_hdr); - mlx5_core_err(mdev, - "fail to alloc psp copy modify_header_id err=%d\n", err); - goto out_spec; + err = accel_psp_fs_create_miss_group(check->ft, &check->drop_group); + if (err) { + mlx5_core_err(fs->mdev, + "fail to create psp rx check drop group err=%d\n", + err); + goto out_err; } accel_psp_setup_syndrome_match(spec, PSP_OK); /* create fte */ - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | - MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; - flow_act.modify_hdr = modify_hdr; - dest[0].type = fs_prot->default_dest.type; - dest[0].ft = fs_prot->default_dest.ft; - dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; - dest[1].counter = fs->rx_fs->rx_counter; - fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, dest, 2); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = fs->rx.ft; + fte = mlx5_add_flow_rules(check->ft, spec, &flow_act, &dest, 1); if (IS_ERR(fte)) { err = PTR_ERR(fte); - mlx5_core_err(mdev, "fail to add psp rx err copy rule err=%d\n", err); - goto out; + mlx5_core_err(mdev, "fail to add psp rx check ok rule err=%d\n", + err); + goto out_err; } - rx_err->rule = fte; + check->rule = fte; /* add auth fail drop rule */ memset(spec, 0, sizeof(*spec)); - memset(&flow_act, 0, sizeof(flow_act)); accel_psp_setup_syndrome_match(spec, PSP_ICV_FAIL); - /* create fte */ - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | - MLX5_FLOW_CONTEXT_ACTION_COUNT; - dest[0].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; - dest[0].counter = fs->rx_fs->rx_auth_fail_counter; - fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, dest, 1); - if (IS_ERR(fte)) { - err = PTR_ERR(fte); - mlx5_core_err(mdev, "fail to add psp rx auth fail drop rule err=%d\n", + err = accel_psp_add_drop_rule(check->ft, spec, + fs->rx_auth_fail_counter, + &check->auth_fail_rule); + if (err) { + mlx5_core_err(mdev, + "fail to add psp rx check auth fail drop rule err=%d\n", err); - goto out_drop_rule; + goto out_err; } - rx_err->auth_fail_rule = fte; /* add framing drop rule */ memset(spec, 0, sizeof(*spec)); - memset(&flow_act, 0, sizeof(flow_act)); accel_psp_setup_syndrome_match(spec, PSP_BAD_TRAILER); - /* create fte */ - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | - MLX5_FLOW_CONTEXT_ACTION_COUNT; - dest[0].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; - dest[0].counter = fs->rx_fs->rx_err_counter; - fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, dest, 1); - if (IS_ERR(fte)) { - err = PTR_ERR(fte); - mlx5_core_err(mdev, "fail to add psp rx framing err drop rule err=%d\n", + err = accel_psp_add_drop_rule(check->ft, spec, fs->rx_err_counter, + &check->err_rule); + if (err) { + mlx5_core_err(mdev, + "fail to add psp rx check framing drop rule err=%d\n", err); - goto out_drop_auth_fail_rule; + goto out_err; } - rx_err->err_rule = fte; /* add misc. errors drop rule */ memset(spec, 0, sizeof(*spec)); - memset(&flow_act, 0, sizeof(flow_act)); - /* create fte */ - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | - MLX5_FLOW_CONTEXT_ACTION_COUNT; - dest[0].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; - dest[0].counter = fs->rx_fs->rx_bad_counter; - fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, dest, 1); - if (IS_ERR(fte)) { - err = PTR_ERR(fte); - mlx5_core_err(mdev, "fail to add psp rx misc. err drop rule err=%d\n", + err = accel_psp_add_drop_rule(check->ft, spec, fs->rx_bad_counter, + &check->bad_rule); + if (err) { + mlx5_core_err(mdev, + "fail to add psp rx check misc. err drop rule err=%d\n", err); - goto out_drop_error_rule; + goto out_err; } - rx_err->bad_rule = fte; - - rx_err->copy_modify_hdr = modify_hdr; goto out_spec; -out_drop_error_rule: - mlx5_del_flow_rules(rx_err->err_rule); - rx_err->err_rule = NULL; -out_drop_auth_fail_rule: - mlx5_del_flow_rules(rx_err->auth_fail_rule); - rx_err->auth_fail_rule = NULL; -out_drop_rule: - mlx5_del_flow_rules(rx_err->rule); - rx_err->rule = NULL; -out: - mlx5_modify_header_dealloc(mdev, modify_hdr); +out_err: + accel_psp_fs_rx_check_ft_destroy(check); out_spec: kfree(spec); return err; } -static int accel_psp_fs_rx_err_create_ft(struct mlx5e_psp_fs *fs, - struct mlx5e_accel_fs_psp_prot *fs_prot, - struct mlx5e_psp_rx_err *rx_err) +static void +accel_psp_fs_rx_decrypt_ft_destroy(struct mlx5e_psp_fs *fs, + struct mlx5e_psp_rx_decrypt_table *decrypt) { - struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs->fs, false); - struct mlx5_flow_table_attr ft_attr = {}; - struct mlx5_flow_table *ft; - int err; - - ft_attr.max_fte = 2; - ft_attr.autogroup.max_num_groups = 2; - ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; // MLX5E_ACCEL_FS_TCP_FT_LEVEL - ft_attr.prio = MLX5E_NIC_PRIO; - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); - if (IS_ERR(ft)) { - err = PTR_ERR(ft); - mlx5_core_err(fs->mdev, "fail to create psp rx inline ft err=%d\n", err); - return err; - } - - rx_err->ft = ft; - err = accel_psp_fs_rx_err_add_rule(fs, fs_prot, rx_err); - if (err) - goto out_err; - - return 0; - -out_err: - mlx5_destroy_flow_table(ft); - rx_err->ft = NULL; - return err; -} - -static void accel_psp_fs_rx_fs_destroy(struct mlx5e_psp_fs *fs, - struct mlx5e_accel_fs_psp_prot *fs_prot) -{ - if (fs_prot->def_rule) { - mlx5_del_flow_rules(fs_prot->def_rule); - fs_prot->def_rule = NULL; - } - - if (fs_prot->rx_modify_hdr) { - mlx5_modify_header_dealloc(fs->mdev, fs_prot->rx_modify_hdr); - fs_prot->rx_modify_hdr = NULL; - } - - if (fs_prot->miss_rule) { - mlx5_del_flow_rules(fs_prot->miss_rule); - fs_prot->miss_rule = NULL; - } - - if (fs_prot->miss_group) { - mlx5_destroy_flow_group(fs_prot->miss_group); - fs_prot->miss_group = NULL; - } - - if (fs_prot->ft) { - mlx5_destroy_flow_table(fs_prot->ft); - fs_prot->ft = NULL; + accel_psp_fs_del_flow_rule(&decrypt->rule); + if (decrypt->rx_modify_hdr) { + mlx5_modify_header_dealloc(fs->mdev, decrypt->rx_modify_hdr); + decrypt->rx_modify_hdr = NULL; } + accel_psp_fs_del_flow_rule(&decrypt->miss_rule); + accel_psp_fs_destroy_flow_group(&decrypt->miss_group); + accel_psp_fs_destroy_ft(&decrypt->ft); } static void setup_fte_udp_psp(struct mlx5_flow_spec *spec, u16 udp_port) @@ -325,70 +425,64 @@ static void setup_fte_udp_psp(struct mlx5_flow_spec *spec, u16 udp_port) MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, IPPROTO_UDP); } -static int accel_psp_fs_rx_create_ft(struct mlx5e_psp_fs *fs, - struct mlx5e_accel_fs_psp_prot *fs_prot) +static int +accel_psp_fs_rx_decrypt_ft_create(struct mlx5e_psp_fs *fs, + struct mlx5e_psp_rx_decrypt_table *decrypt, + struct mlx5_flow_destination *default_dest) { - struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs->fs, false); u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; - int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_modify_hdr *modify_hdr = NULL; struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_destination dest = {}; struct mlx5_core_dev *mdev = fs->mdev; - struct mlx5_flow_group *miss_group; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; - struct mlx5_flow_table *ft; - u32 *flow_group_in; int err = 0; - flow_group_in = kvzalloc(inlen, GFP_KERNEL); spec = kvzalloc_obj(*spec); - if (!flow_group_in || !spec) { - err = -ENOMEM; - goto out; - } + if (!spec) + return -ENOMEM; /* Create FT */ ft_attr.max_fte = 2; - ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; - ft_attr.prio = MLX5E_NIC_PRIO; + ft_attr.level = MLX5E_ACCEL_FS_PSP_FT_LEVEL; ft_attr.autogroup.num_reserved_entries = 1; ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); - if (IS_ERR(ft)) { - err = PTR_ERR(ft); - mlx5_core_err(mdev, "fail to create psp rx ft err=%d\n", err); + ft_attr.prio = MLX5E_NIC_PRIO; + err = accel_psp_fs_create_ft(fs, &ft_attr, &decrypt->ft); + if (err) { + mlx5_core_err(mdev, "fail to create psp rx decrypt ft err=%d\n", + err); goto out_err; } - fs_prot->ft = ft; /* Create miss_group */ - MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); - MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); - miss_group = mlx5_create_flow_group(ft, flow_group_in); - if (IS_ERR(miss_group)) { - err = PTR_ERR(miss_group); - mlx5_core_err(mdev, "fail to create psp rx miss_group err=%d\n", err); + err = accel_psp_fs_create_miss_group(decrypt->ft, &decrypt->miss_group); + if (err) { + mlx5_core_err(mdev, + "fail to create psp rx decrypt miss_group err=%d\n", + err); goto out_err; } - fs_prot->miss_group = miss_group; /* Create miss rule */ - rule = mlx5_add_flow_rules(ft, spec, &flow_act, &fs_prot->default_dest, 1); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + rule = mlx5_add_flow_rules(decrypt->ft, spec, &flow_act, default_dest, + 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - mlx5_core_err(mdev, "fail to create psp rx miss_rule err=%d\n", err); + mlx5_core_err(mdev, + "fail to create psp rx decrypt miss_rule err=%d\n", + err); goto out_err; } - fs_prot->miss_rule = rule; + decrypt->miss_rule = rule; - /* Add default Rx psp rule */ + /* Add PSP RX decrypt rule */ setup_fte_udp_psp(spec, PSP_DEFAULT_UDP_PORT); flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_PSP; /* Set bit[31, 30] PSP marker */ - /* Set bit[29-23] psp_syndrome is set in error FT */ #define MLX5E_PSP_MARKER_BIT (BIT(30) | BIT(31)) MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); @@ -403,275 +497,159 @@ static int accel_psp_fs_rx_create_ft(struct mlx5e_psp_fs *fs, modify_hdr = NULL; goto out_err; } - fs_prot->rx_modify_hdr = modify_hdr; + decrypt->rx_modify_hdr = modify_hdr; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; flow_act.modify_hdr = modify_hdr; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = fs_prot->rx_err.ft; - rule = mlx5_add_flow_rules(fs_prot->ft, spec, &flow_act, &dest, 1); + dest.ft = fs->check.ft; + rule = mlx5_add_flow_rules(decrypt->ft, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - mlx5_core_err(mdev, - "fail to add psp rule Rx decryption, err=%d, flow_act.action = %#04X\n", - err, flow_act.action); + mlx5_core_err(mdev, "fail to add psp rx decrypt rule, err=%d\n", + err); goto out_err; } - fs_prot->def_rule = rule; - goto out; + decrypt->rule = rule; + goto out_spec; out_err: - accel_psp_fs_rx_fs_destroy(fs, fs_prot); -out: - kvfree(flow_group_in); + accel_psp_fs_rx_decrypt_ft_destroy(fs, decrypt); +out_spec: kvfree(spec); return err; } -static int accel_psp_fs_rx_destroy(struct mlx5e_psp_fs *fs, enum accel_fs_psp_type type) -{ - struct mlx5e_accel_fs_psp_prot *fs_prot; - struct mlx5e_accel_fs_psp *accel_psp; - - accel_psp = fs->rx_fs; - - /* The netdev unreg already happened, so all offloaded rule are already removed */ - fs_prot = &accel_psp->fs_prot[type]; - - accel_psp_fs_rx_fs_destroy(fs, fs_prot); - - accel_psp_fs_rx_err_destroy_ft(fs, &fs_prot->rx_err); - - return 0; -} - -static int accel_psp_fs_rx_create(struct mlx5e_psp_fs *fs, enum accel_fs_psp_type type) +static void accel_psp_fs_rx_destroy(struct mlx5e_psp_fs *fs) { struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false); - struct mlx5e_accel_fs_psp_prot *fs_prot; - struct mlx5e_accel_fs_psp *accel_psp; - int err; - - accel_psp = fs->rx_fs; - fs_prot = &accel_psp->fs_prot[type]; - - fs_prot->default_dest = mlx5_ttc_get_default_dest(ttc, fs_psp2tt(type)); - - err = accel_psp_fs_rx_err_create_ft(fs, fs_prot, &fs_prot->rx_err); - if (err) - return err; - - err = accel_psp_fs_rx_create_ft(fs, fs_prot); - if (err) - accel_psp_fs_rx_err_destroy_ft(fs, &fs_prot->rx_err); - - return err; -} - -static int accel_psp_fs_rx_ft_get(struct mlx5e_psp_fs *fs, enum accel_fs_psp_type type) -{ - struct mlx5e_accel_fs_psp_prot *fs_prot; - struct mlx5_flow_destination dest = {}; - struct mlx5e_accel_fs_psp *accel_psp; - struct mlx5_ttc_table *ttc; - int err = 0; - - if (!fs || !fs->rx_fs) - return -EINVAL; - - ttc = mlx5e_fs_get_ttc(fs->fs, false); - accel_psp = fs->rx_fs; - fs_prot = &accel_psp->fs_prot[type]; - mutex_lock(&fs_prot->prot_mutex); - if (fs_prot->refcnt++) - goto out; - - /* create FT */ - err = accel_psp_fs_rx_create(fs, type); - if (err) { - fs_prot->refcnt--; - goto out; - } - - /* connect */ - dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = fs_prot->ft; - mlx5_ttc_fwd_dest(ttc, fs_psp2tt(type), &dest); - -out: - mutex_unlock(&fs_prot->prot_mutex); - return err; -} - -static void accel_psp_fs_rx_ft_put(struct mlx5e_psp_fs *fs, enum accel_fs_psp_type type) -{ - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false); - struct mlx5e_accel_fs_psp_prot *fs_prot; - struct mlx5e_accel_fs_psp *accel_psp; - - accel_psp = fs->rx_fs; - fs_prot = &accel_psp->fs_prot[type]; - mutex_lock(&fs_prot->prot_mutex); - if (--fs_prot->refcnt) - goto out; + int i; /* disconnect */ - mlx5_ttc_fwd_default_dest(ttc, fs_psp2tt(type)); - - /* remove FT */ - accel_psp_fs_rx_destroy(fs, type); - -out: - mutex_unlock(&fs_prot->prot_mutex); + for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { + mlx5_ttc_fwd_default_dest(ttc, fs_psp2tt(i)); + accel_psp_fs_rx_decrypt_ft_destroy(fs, &fs->decrypt[i]); + } + accel_psp_fs_rx_check_ft_destroy(&fs->check); + accel_psp_fs_rx_ft_destroy(&fs->rx); } -static void accel_psp_fs_cleanup_rx(struct mlx5e_psp_fs *fs) +static int accel_psp_fs_rx_create(struct mlx5e_psp_fs *fs, + struct netlink_ext_ack *extack) { - struct mlx5e_accel_fs_psp_prot *fs_prot; - struct mlx5e_accel_fs_psp *accel_psp; - enum accel_fs_psp_type i; + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false); + int i, err; - if (!fs->rx_fs) - return; - - accel_psp = fs->rx_fs; - mlx5_fc_destroy(fs->mdev, accel_psp->rx_bad_counter); - mlx5_fc_destroy(fs->mdev, accel_psp->rx_err_counter); - mlx5_fc_destroy(fs->mdev, accel_psp->rx_auth_fail_counter); - mlx5_fc_destroy(fs->mdev, accel_psp->rx_counter); - for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { - fs_prot = &accel_psp->fs_prot[i]; - mutex_destroy(&fs_prot->prot_mutex); - WARN_ON(fs_prot->refcnt); + err = accel_psp_fs_rx_ft_create(fs, &fs->rx); + if (err) { + NL_SET_ERR_MSG(extack, "Failed creating RX steering table"); + return err; } - kfree(fs->rx_fs); - fs->rx_fs = NULL; -} -static int accel_psp_fs_init_rx(struct mlx5e_psp_fs *fs) -{ - struct mlx5e_accel_fs_psp_prot *fs_prot; - struct mlx5e_accel_fs_psp *accel_psp; - struct mlx5_core_dev *mdev = fs->mdev; - struct mlx5_fc *flow_counter; - enum accel_fs_psp_type i; - int err; - - accel_psp = kzalloc_obj(*accel_psp); - if (!accel_psp) - return -ENOMEM; + err = accel_psp_fs_rx_check_ft_create(fs, &fs->check); + if (err) { + NL_SET_ERR_MSG(extack, + "Failed creating RX check steering table"); + goto err_ft; + } for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { - fs_prot = &accel_psp->fs_prot[i]; - mutex_init(&fs_prot->prot_mutex); - } + struct mlx5_flow_destination dest; - flow_counter = mlx5_fc_create(mdev, false); - if (IS_ERR(flow_counter)) { - mlx5_core_warn(mdev, - "fail to create psp rx flow counter err=%pe\n", - flow_counter); - err = PTR_ERR(flow_counter); - goto out_err; - } - accel_psp->rx_counter = flow_counter; + dest = mlx5_ttc_get_default_dest(ttc, fs_psp2tt(i)); + err = accel_psp_fs_rx_decrypt_ft_create(fs, &fs->decrypt[i], + &dest); + if (err) { + NL_SET_ERR_MSG(extack, + "Failed creating RX decrypt steering table"); + goto err_decrypt_ft; + } - flow_counter = mlx5_fc_create(mdev, false); - if (IS_ERR(flow_counter)) { - mlx5_core_warn(mdev, - "fail to create psp rx auth fail flow counter err=%pe\n", - flow_counter); - err = PTR_ERR(flow_counter); - goto out_counter_err; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = fs->decrypt[i].ft; + mlx5_ttc_fwd_dest(ttc, fs_psp2tt(i), &dest); } - accel_psp->rx_auth_fail_counter = flow_counter; - - flow_counter = mlx5_fc_create(mdev, false); - if (IS_ERR(flow_counter)) { - mlx5_core_warn(mdev, - "fail to create psp rx error flow counter err=%pe\n", - flow_counter); - err = PTR_ERR(flow_counter); - goto out_auth_fail_counter_err; - } - accel_psp->rx_err_counter = flow_counter; - - flow_counter = mlx5_fc_create(mdev, false); - if (IS_ERR(flow_counter)) { - mlx5_core_warn(mdev, - "fail to create psp rx bad flow counter err=%pe\n", - flow_counter); - err = PTR_ERR(flow_counter); - goto out_err_counter_err; - } - accel_psp->rx_bad_counter = flow_counter; - - fs->rx_fs = accel_psp; return 0; -out_err_counter_err: - mlx5_fc_destroy(mdev, accel_psp->rx_err_counter); - accel_psp->rx_err_counter = NULL; -out_auth_fail_counter_err: - mlx5_fc_destroy(mdev, accel_psp->rx_auth_fail_counter); - accel_psp->rx_auth_fail_counter = NULL; -out_counter_err: - mlx5_fc_destroy(mdev, accel_psp->rx_counter); - accel_psp->rx_counter = NULL; -out_err: - for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { - fs_prot = &accel_psp->fs_prot[i]; - mutex_destroy(&fs_prot->prot_mutex); +err_decrypt_ft: + while (--i >= 0) { + mlx5_ttc_fwd_default_dest(ttc, fs_psp2tt(i)); + accel_psp_fs_rx_decrypt_ft_destroy(fs, &fs->decrypt[i]); } - kfree(accel_psp); - fs->rx_fs = NULL; + accel_psp_fs_rx_check_ft_destroy(&fs->check); +err_ft: + accel_psp_fs_rx_ft_destroy(&fs->rx); + return err; +} +static void accel_psp_fs_rx_cleanup(struct mlx5e_psp_fs *fs) +{ + accel_psp_fs_destroy_counter(fs->mdev, &fs->rx_bad_counter); + accel_psp_fs_destroy_counter(fs->mdev, &fs->rx_err_counter); + accel_psp_fs_destroy_counter(fs->mdev, &fs->rx_auth_fail_counter); + accel_psp_fs_destroy_counter(fs->mdev, &fs->rx_counter); +} + +static int accel_psp_fs_rx_init(struct mlx5e_psp_fs *fs) +{ + struct mlx5_core_dev *mdev = fs->mdev; + int err; + + err = accel_psp_fs_create_counter(mdev, &fs->rx_counter); + if (err) { + mlx5_core_warn(mdev, + "fail to create psp rx flow counter err=%d\n", + err); + goto out_err; + } + + err = accel_psp_fs_create_counter(mdev, &fs->rx_auth_fail_counter); + if (err) { + mlx5_core_warn(mdev, + "fail to create psp rx auth fail flow counter err=%d\n", + err); + goto out_err; + } + + err = accel_psp_fs_create_counter(mdev, &fs->rx_err_counter); + if (err) { + mlx5_core_warn(mdev, + "fail to create psp rx error flow counter err=%d\n", + err); + goto out_err; + } + + err = accel_psp_fs_create_counter(mdev, &fs->rx_bad_counter); + if (err) { + mlx5_core_warn(mdev, + "fail to create psp rx bad flow counter err=%d\n", + err); + goto out_err; + } + + return 0; + +out_err: + accel_psp_fs_rx_cleanup(fs); return err; } void mlx5_accel_psp_fs_cleanup_rx_tables(struct mlx5e_priv *priv) { - int i; - if (!priv->psp) return; - for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) - accel_psp_fs_rx_ft_put(priv->psp->fs, i); + netdev_lock(priv->netdev); + accel_psp_fs_rx_destroy(priv->psp->fs); + netdev_unlock(priv->netdev); } -int mlx5_accel_psp_fs_init_rx_tables(struct mlx5e_priv *priv) -{ - struct mlx5e_psp_fs *fs; - int err, i; - - if (!priv->psp) - return 0; - - fs = priv->psp->fs; - for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) { - err = accel_psp_fs_rx_ft_get(fs, i); - if (err) - goto out_err; - } - - return 0; - -out_err: - i--; - while (i >= 0) { - accel_psp_fs_rx_ft_put(fs, i); - --i; - } - - return err; -} - -static int accel_psp_fs_tx_create_ft_table(struct mlx5e_psp_fs *fs) +static int accel_psp_fs_tx_ft_create(struct mlx5e_psp_fs *fs, + struct mlx5e_psp_tx_table *tx) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_table_attr ft_attr = {}; @@ -681,7 +659,6 @@ static int accel_psp_fs_tx_create_ft_table(struct mlx5e_psp_fs *fs) u32 *in, *mc, *outer_headers_c; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; - struct mlx5e_psp_tx *tx_fs; struct mlx5_flow_table *ft; struct mlx5_flow_group *fg; int err = 0; @@ -700,8 +677,7 @@ static int accel_psp_fs_tx_create_ft_table(struct mlx5e_psp_fs *fs) ft_attr.level = MLX5E_PSP_LEVEL; ft_attr.autogroup.max_num_groups = 1; - tx_fs = fs->tx_fs; - ft = mlx5_create_flow_table(tx_fs->ns, &ft_attr); + ft = mlx5_create_flow_table(tx->ns, &ft_attr); if (IS_ERR(ft)) { err = PTR_ERR(ft); mlx5_core_err(mdev, "PSP: fail to add psp tx flow table, err = %d\n", err); @@ -727,7 +703,7 @@ static int accel_psp_fs_tx_create_ft_table(struct mlx5e_psp_fs *fs) MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | MLX5_FLOW_CONTEXT_ACTION_COUNT; dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; - dest.counter = tx_fs->tx_counter; + dest.counter = fs->tx_counter; rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -735,9 +711,9 @@ static int accel_psp_fs_tx_create_ft_table(struct mlx5e_psp_fs *fs) goto err_add_flow_rule; } - tx_fs->ft = ft; - tx_fs->fg = fg; - tx_fs->rule = rule; + tx->ft = ft; + tx->fg = fg; + tx->rule = rule; goto out; err_add_flow_rule: @@ -750,87 +726,35 @@ static int accel_psp_fs_tx_create_ft_table(struct mlx5e_psp_fs *fs) return err; } -static void accel_psp_fs_tx_destroy(struct mlx5e_psp_tx *tx_fs) +static void accel_psp_fs_tx_ft_destroy(struct mlx5e_psp_tx_table *tx) { - if (!tx_fs->ft) - return; - - mlx5_del_flow_rules(tx_fs->rule); - mlx5_destroy_flow_group(tx_fs->fg); - mlx5_destroy_flow_table(tx_fs->ft); + accel_psp_fs_del_flow_rule(&tx->rule); + accel_psp_fs_destroy_flow_group(&tx->fg); + accel_psp_fs_destroy_ft(&tx->ft); } -static int accel_psp_fs_tx_ft_get(struct mlx5e_psp_fs *fs) +static void accel_psp_fs_tx_cleanup(struct mlx5e_psp_fs *fs) { - struct mlx5e_psp_tx *tx_fs = fs->tx_fs; - int err = 0; - - mutex_lock(&tx_fs->mutex); - if (tx_fs->refcnt++) - goto out; - - err = accel_psp_fs_tx_create_ft_table(fs); - if (err) - tx_fs->refcnt--; -out: - mutex_unlock(&tx_fs->mutex); - return err; + accel_psp_fs_destroy_counter(fs->mdev, &fs->tx_counter); } -static void accel_psp_fs_tx_ft_put(struct mlx5e_psp_fs *fs) -{ - struct mlx5e_psp_tx *tx_fs = fs->tx_fs; - - mutex_lock(&tx_fs->mutex); - if (--tx_fs->refcnt) - goto out; - - accel_psp_fs_tx_destroy(tx_fs); -out: - mutex_unlock(&tx_fs->mutex); -} - -static void accel_psp_fs_cleanup_tx(struct mlx5e_psp_fs *fs) -{ - struct mlx5e_psp_tx *tx_fs = fs->tx_fs; - - if (!tx_fs) - return; - - mlx5_fc_destroy(fs->mdev, tx_fs->tx_counter); - mutex_destroy(&tx_fs->mutex); - WARN_ON(tx_fs->refcnt); - kfree(tx_fs); - fs->tx_fs = NULL; -} - -static int accel_psp_fs_init_tx(struct mlx5e_psp_fs *fs) +static int accel_psp_fs_tx_init(struct mlx5e_psp_fs *fs) { struct mlx5_core_dev *mdev = fs->mdev; - struct mlx5_flow_namespace *ns; - struct mlx5_fc *flow_counter; - struct mlx5e_psp_tx *tx_fs; + int err; - ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); - if (!ns) + fs->tx.ns = mlx5_get_flow_namespace(mdev, + MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); + if (!fs->tx.ns) return -EOPNOTSUPP; - tx_fs = kzalloc_obj(*tx_fs); - if (!tx_fs) - return -ENOMEM; - - flow_counter = mlx5_fc_create(mdev, false); - if (IS_ERR(flow_counter)) { + err = accel_psp_fs_create_counter(mdev, &fs->tx_counter); + if (err) { mlx5_core_warn(mdev, - "fail to create psp tx flow counter err=%pe\n", - flow_counter); - kfree(tx_fs); - return PTR_ERR(flow_counter); + "fail to create psp tx flow counter err=%d\n", + err); + return err; } - tx_fs->tx_counter = flow_counter; - mutex_init(&tx_fs->mutex); - tx_fs->ns = ns; - fs->tx_fs = tx_fs; return 0; } @@ -838,32 +762,29 @@ static void mlx5e_accel_psp_fs_get_stats_fill(struct mlx5e_priv *priv, struct mlx5e_psp_stats *stats) { - struct mlx5e_psp_tx *tx_fs = priv->psp->fs->tx_fs; + struct mlx5e_psp_fs *fs = priv->psp->fs; struct mlx5_core_dev *mdev = priv->mdev; - struct mlx5e_accel_fs_psp *accel_psp; - accel_psp = (struct mlx5e_accel_fs_psp *)priv->psp->fs->rx_fs; - - if (tx_fs->tx_counter) - mlx5_fc_query(mdev, tx_fs->tx_counter, &stats->psp_tx_pkts, + if (fs->tx_counter) + mlx5_fc_query(mdev, fs->tx_counter, &stats->psp_tx_pkts, &stats->psp_tx_bytes); - if (accel_psp->rx_counter) - mlx5_fc_query(mdev, accel_psp->rx_counter, &stats->psp_rx_pkts, + if (fs->rx_counter) + mlx5_fc_query(mdev, fs->rx_counter, &stats->psp_rx_pkts, &stats->psp_rx_bytes); - if (accel_psp->rx_auth_fail_counter) - mlx5_fc_query(mdev, accel_psp->rx_auth_fail_counter, + if (fs->rx_auth_fail_counter) + mlx5_fc_query(mdev, fs->rx_auth_fail_counter, &stats->psp_rx_pkts_auth_fail, &stats->psp_rx_bytes_auth_fail); - if (accel_psp->rx_err_counter) - mlx5_fc_query(mdev, accel_psp->rx_err_counter, + if (fs->rx_err_counter) + mlx5_fc_query(mdev, fs->rx_err_counter, &stats->psp_rx_pkts_frame_err, &stats->psp_rx_bytes_frame_err); - if (accel_psp->rx_bad_counter) - mlx5_fc_query(mdev, accel_psp->rx_bad_counter, + if (fs->rx_bad_counter) + mlx5_fc_query(mdev, fs->rx_bad_counter, &stats->psp_rx_pkts_drop, &stats->psp_rx_bytes_drop); } @@ -873,21 +794,15 @@ void mlx5_accel_psp_fs_cleanup_tx_tables(struct mlx5e_priv *priv) if (!priv->psp) return; - accel_psp_fs_tx_ft_put(priv->psp->fs); -} - -int mlx5_accel_psp_fs_init_tx_tables(struct mlx5e_priv *priv) -{ - if (!priv->psp) - return 0; - - return accel_psp_fs_tx_ft_get(priv->psp->fs); + netdev_lock(priv->netdev); + accel_psp_fs_tx_ft_destroy(&priv->psp->fs->tx); + netdev_unlock(priv->netdev); } static void mlx5e_accel_psp_fs_cleanup(struct mlx5e_psp_fs *fs) { - accel_psp_fs_cleanup_rx(fs); - accel_psp_fs_cleanup_tx(fs); + accel_psp_fs_rx_cleanup(fs); + accel_psp_fs_tx_cleanup(fs); kfree(fs); } @@ -901,29 +816,63 @@ static struct mlx5e_psp_fs *mlx5e_accel_psp_fs_init(struct mlx5e_priv *priv) return ERR_PTR(-ENOMEM); fs->mdev = priv->mdev; - err = accel_psp_fs_init_tx(fs); + err = accel_psp_fs_tx_init(fs); if (err) goto err_tx; fs->fs = priv->fs; - err = accel_psp_fs_init_rx(fs); + err = accel_psp_fs_rx_init(fs); if (err) goto err_rx; return fs; err_rx: - accel_psp_fs_cleanup_tx(fs); + accel_psp_fs_tx_cleanup(fs); err_tx: kfree(fs); return ERR_PTR(err); } +static int accel_psp_fs_create(struct mlx5e_priv *priv, + struct netlink_ext_ack *extack) +{ + int err; + + err = accel_psp_fs_rx_create(priv->psp->fs, extack); + if (err) + return err; + + err = accel_psp_fs_tx_ft_create(priv->psp->fs, &priv->psp->fs->tx); + if (err) { + NL_SET_ERR_MSG(extack, "Failed creating TX steering table"); + accel_psp_fs_rx_destroy(priv->psp->fs); + } + return err; +} + +static void accel_psp_fs_destroy(struct mlx5e_priv *priv) +{ + accel_psp_fs_tx_ft_destroy(&priv->psp->fs->tx); + accel_psp_fs_rx_destroy(priv->psp->fs); +} + static int mlx5e_psp_set_config(struct psp_dev *psd, struct psp_dev_config *conf, struct netlink_ext_ack *extack) { - return 0; /* TODO: this should actually do things to the device */ + struct mlx5e_priv *priv = netdev_priv(psd->main_netdev); + bool psp_enabled = psd->config.versions; + bool enable_psp = conf->versions; + int err = 0; + + netdev_lock(priv->netdev); + if (!psp_enabled && enable_psp) + err = accel_psp_fs_create(priv, extack); + else if (psp_enabled && !enable_psp) + accel_psp_fs_destroy(priv); + netdev_unlock(priv->netdev); + return err; } static int @@ -1072,21 +1021,21 @@ void mlx5e_psp_unregister(struct mlx5e_priv *priv) { struct mlx5e_psp *psp = priv->psp; - if (!psp || !psp->psp) + if (!psp || !psp->psd) return; - psp_dev_unregister(psp->psp); - psp->psp = NULL; + psp_dev_unregister(psp->psd); + psp->psd = NULL; } -void mlx5e_psp_register(struct mlx5e_priv *priv) +int mlx5e_psp_register(struct mlx5e_priv *priv) { struct mlx5e_psp *psp = priv->psp; struct psp_dev *psd; /* FW Caps missing */ if (!priv->psp) - return; + return 0; psp->caps.assoc_drv_spc = sizeof(u32); psp->caps.versions = 1 << PSP_VERSION_HDR0_AES_GCM_128; @@ -1098,9 +1047,11 @@ void mlx5e_psp_register(struct mlx5e_priv *priv) if (IS_ERR(psd)) { mlx5_core_err(priv->mdev, "PSP failed to register due to %pe\n", psd); - return; + return PTR_ERR(psd); } - psp->psp = psd; + psp->psd = psd; + + return 0; } int mlx5e_psp_init(struct mlx5e_priv *priv) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h index 6b62fef0d9a7..3f441e7dd55a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp.h @@ -23,7 +23,7 @@ struct mlx5e_psp_stats { }; struct mlx5e_psp { - struct psp_dev *psp; + struct psp_dev *psd; struct psp_dev_caps caps; struct mlx5e_psp_fs *fs; atomic_t tx_key_cnt; @@ -43,33 +43,21 @@ static inline bool mlx5_is_psp_device(struct mlx5_core_dev *mdev) return true; } -int mlx5_accel_psp_fs_init_rx_tables(struct mlx5e_priv *priv); void mlx5_accel_psp_fs_cleanup_rx_tables(struct mlx5e_priv *priv); -int mlx5_accel_psp_fs_init_tx_tables(struct mlx5e_priv *priv); void mlx5_accel_psp_fs_cleanup_tx_tables(struct mlx5e_priv *priv); -void mlx5e_psp_register(struct mlx5e_priv *priv); +int mlx5e_psp_register(struct mlx5e_priv *priv); void mlx5e_psp_unregister(struct mlx5e_priv *priv); int mlx5e_psp_init(struct mlx5e_priv *priv); void mlx5e_psp_cleanup(struct mlx5e_priv *priv); #else -static inline int mlx5_accel_psp_fs_init_rx_tables(struct mlx5e_priv *priv) -{ - return 0; -} - static inline void mlx5_accel_psp_fs_cleanup_rx_tables(struct mlx5e_priv *priv) { } -static inline int mlx5_accel_psp_fs_init_tx_tables(struct mlx5e_priv *priv) -{ - return 0; -} - static inline void mlx5_accel_psp_fs_cleanup_tx_tables(struct mlx5e_priv *priv) { } static inline bool mlx5_is_psp_device(struct mlx5_core_dev *mdev) { return false; } -static inline void mlx5e_psp_register(struct mlx5e_priv *priv) { } +static inline int mlx5e_psp_register(struct mlx5e_priv *priv) { return 0; } static inline void mlx5e_psp_unregister(struct mlx5e_priv *priv) { } static inline int mlx5e_psp_init(struct mlx5e_priv *priv) { return 0; } static inline void mlx5e_psp_cleanup(struct mlx5e_priv *priv) { } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c index ef7f5338540f..348fd7a96261 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.c @@ -14,12 +14,6 @@ #include "en_accel/psp_rxtx.h" #include "en_accel/psp.h" -enum { - MLX5E_PSP_OFFLOAD_RX_SYNDROME_DECRYPTED, - MLX5E_PSP_OFFLOAD_RX_SYNDROME_AUTH_FAILED, - MLX5E_PSP_OFFLOAD_RX_SYNDROME_BAD_TRAILER, -}; - static void mlx5e_psp_set_swp(struct sk_buff *skb, struct mlx5e_accel_tx_psp_state *psp_st, struct mlx5_wqe_eth_seg *eseg) @@ -122,16 +116,11 @@ static bool mlx5e_psp_set_state(struct mlx5e_priv *priv, bool mlx5e_psp_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb, struct mlx5_cqe64 *cqe) { - u32 psp_meta_data = be32_to_cpu(cqe->ft_metadata); struct mlx5e_priv *priv = netdev_priv(netdev); - u16 dev_id = priv->psp->psp->id; + u16 dev_id = priv->psp->psd->id; bool strip_icv = true; u8 generation = 0; - /* TBD: report errors as SW counters to ethtool, any further handling ? */ - if (MLX5_PSP_METADATA_SYNDROME(psp_meta_data) != MLX5E_PSP_OFFLOAD_RX_SYNDROME_DECRYPTED) - goto drop; - if (psp_dev_rcv(skb, dev_id, generation, strip_icv)) goto drop; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.h index 70289c921bd6..2b080c39cc37 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/psp_rxtx.h @@ -10,9 +10,8 @@ #include "en.h" #include "en/txrx.h" -/* Bit30: PSP marker, Bit29-23: PSP syndrome, Bit22-0: PSP obj id */ +/* Bit30: PSP marker, Bit22-0: PSP obj id */ #define MLX5_PSP_METADATA_MARKER(metadata) ((((metadata) >> 30) & 0x3) == 0x3) -#define MLX5_PSP_METADATA_SYNDROME(metadata) (((metadata) >> 23) & GENMASK(6, 0)) #define MLX5_PSP_METADATA_HANDLE(metadata) ((metadata) & GENMASK(22, 0)) struct mlx5e_accel_tx_psp_state { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index aa8610cedaa8..43471c6e6385 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -6193,7 +6193,7 @@ static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) return 0; } -static void mlx5e_nic_enable(struct mlx5e_priv *priv) +static int mlx5e_nic_enable(struct mlx5e_priv *priv) { struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; @@ -6201,7 +6201,9 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) mlx5e_fs_init_l2_addr(priv->fs, netdev); mlx5e_ipsec_init(priv); - mlx5e_psp_register(priv); + err = mlx5e_psp_register(priv); + if (err) + goto out_ipsec_cleanup; err = mlx5e_macsec_init(priv); if (err) @@ -6224,7 +6226,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) mlx5e_pcie_cong_event_init(priv); mlx5e_hv_vhca_stats_create(priv); if (netdev->reg_state != NETREG_REGISTERED) - return; + return 0; mlx5e_dcbnl_init_app(priv); mlx5e_nic_set_rx_mode(priv); @@ -6237,6 +6239,12 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) netdev_unlock(netdev); netif_device_attach(netdev); rtnl_unlock(); + + return 0; + +out_ipsec_cleanup: + mlx5e_ipsec_cleanup(priv); + return err; } static void mlx5e_nic_disable(struct mlx5e_priv *priv) @@ -6618,13 +6626,18 @@ int mlx5e_attach_netdev(struct mlx5e_priv *priv) if (err) goto err_cleanup_tx; - if (profile->enable) - profile->enable(priv); + if (profile->enable) { + err = profile->enable(priv); + if (err) + goto err_cleanup_rx; + } mlx5e_update_features(priv->netdev); return 0; +err_cleanup_rx: + profile->cleanup_rx(priv); err_cleanup_tx: profile->cleanup_tx(priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index c8b76d301c92..603051ab1eaa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -1262,9 +1262,11 @@ static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv) mlx5e_rep_neigh_cleanup(rpriv); } -static void mlx5e_rep_enable(struct mlx5e_priv *priv) +static int mlx5e_rep_enable(struct mlx5e_priv *priv) { mlx5e_set_netdev_mtu_boundaries(priv); + + return 0; } static void mlx5e_rep_disable(struct mlx5e_priv *priv) @@ -1322,7 +1324,7 @@ static int uplink_rep_async_event(struct notifier_block *nb, unsigned long event return NOTIFY_DONE; } -static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) +static int mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) { struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; @@ -1357,6 +1359,8 @@ static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) netdev_unlock(netdev); netif_device_attach(netdev); rtnl_unlock(); + + return 0; } static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv)