diff --git a/drivers/infiniband/hw/mlx5/ib_rep.c b/drivers/infiniband/hw/mlx5/ib_rep.c index 1709b628702e..65d8767d1830 100644 --- a/drivers/infiniband/hw/mlx5/ib_rep.c +++ b/drivers/infiniband/hw/mlx5/ib_rep.c @@ -262,9 +262,10 @@ mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep) struct mlx5_core_dev *peer_mdev; struct mlx5_eswitch *esw; + /* Called while the master E-Switch reps_lock is held. */ mlx5_lag_for_each_peer_mdev(mdev, peer_mdev, i) { esw = peer_mdev->priv.eswitch; - mlx5_eswitch_unregister_vport_reps(esw, REP_IB); + mlx5_eswitch_unregister_vport_reps_nested(esw, REP_IB); } mlx5_ib_release_transport(mdev); } @@ -284,9 +285,10 @@ static void mlx5_ib_register_peer_vport_reps(struct mlx5_core_dev *mdev) struct mlx5_eswitch *esw; int i; + /* Called while the master E-Switch reps_lock is held. */ mlx5_lag_for_each_peer_mdev(mdev, peer_mdev, i) { esw = peer_mdev->priv.eswitch; - mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_IB); + mlx5_eswitch_register_vport_reps_nested(esw, &rep_ops, REP_IB); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 66a773a99876..f70737437954 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1712,6 +1712,7 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) mlx5_lag_disable_change(esw->dev); mlx5_eswitch_invalidate_wq(esw); + mlx5_esw_reps_block(esw); if (!mlx5_esw_is_fdb_created(esw)) { ret = mlx5_eswitch_enable_locked(esw, num_vfs); @@ -1735,6 +1736,8 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) } } + mlx5_esw_reps_unblock(esw); + if (toggle_lag) mlx5_lag_enable_change(esw->dev); @@ -1759,6 +1762,7 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) esw->esw_funcs.num_vfs, esw->esw_funcs.num_ec_vfs, esw->enabled_vports); mlx5_eswitch_invalidate_wq(esw); + mlx5_esw_reps_block(esw); if (!mlx5_core_is_ecpf(esw->dev)) { mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); @@ -1770,6 +1774,8 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) mlx5_eswitch_clear_ec_vf_vports_info(esw); } + mlx5_esw_reps_unblock(esw); + if (esw->mode == MLX5_ESWITCH_OFFLOADS) { struct devlink *devlink = priv_to_devlink(esw->dev); @@ -1825,7 +1831,11 @@ void mlx5_eswitch_disable(struct mlx5_eswitch *esw) devl_assert_locked(priv_to_devlink(esw->dev)); mlx5_lag_disable_change(esw->dev); + + mlx5_esw_reps_block(esw); mlx5_eswitch_disable_locked(esw); + mlx5_esw_reps_unblock(esw); + esw->mode = MLX5_ESWITCH_LEGACY; mlx5_lag_enable_change(esw->dev); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 2fd601bd102f..3858690e09b4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -316,6 +316,7 @@ struct mlx5_esw_offload { DECLARE_HASHTABLE(termtbl_tbl, 8); struct mutex termtbl_mutex; /* protects termtbl hash */ struct xarray vhca_map; + struct mutex reps_lock; /* protects representor load/unload/register */ const struct mlx5_eswitch_rep_ops *rep_ops[NUM_REP_TYPES]; u8 inline_mode; atomic64_t num_flows; @@ -951,6 +952,8 @@ mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_handle * mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num, struct mlx5_flow_table *lag_ft); +void mlx5_esw_reps_block(struct mlx5_eswitch *esw); +void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw); #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } @@ -1028,6 +1031,9 @@ mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev) return true; } +static inline void mlx5_esw_reps_block(struct mlx5_eswitch *esw) {} +static inline void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw) {} + static inline bool mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 69ddf56e2fc9..dea5647de548 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "mlx5_core.h" #include "eswitch.h" #include "esw/indir_table.h" @@ -2413,23 +2414,44 @@ static int esw_create_restore_table(struct mlx5_eswitch *esw) return err; } +static void mlx5_esw_assert_reps_locked(struct mlx5_eswitch *esw) +{ + lockdep_assert_held(&esw->offloads.reps_lock); +} + +void mlx5_esw_reps_block(struct mlx5_eswitch *esw) +{ + mutex_lock(&esw->offloads.reps_lock); +} + +static void mlx5_esw_reps_block_nested(struct mlx5_eswitch *esw) +{ + mutex_lock_nested(&esw->offloads.reps_lock, SINGLE_DEPTH_NESTING); +} + +void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw) +{ + mutex_unlock(&esw->offloads.reps_lock); +} + static void esw_mode_change(struct mlx5_eswitch *esw, u16 mode) { + mlx5_esw_reps_unblock(esw); mlx5_devcom_comp_lock(esw->dev->priv.hca_devcom_comp); if (esw->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV || mlx5_core_mp_enabled(esw->dev)) { esw->mode = mode; - mlx5_rescan_drivers_locked(esw->dev); - mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp); - return; + goto out; } esw->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; mlx5_rescan_drivers_locked(esw->dev); esw->mode = mode; esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; +out: mlx5_rescan_drivers_locked(esw->dev); mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp); + mlx5_esw_reps_block(esw); } static void mlx5_esw_fdb_drop_destroy(struct mlx5_eswitch *esw) @@ -2645,6 +2667,7 @@ static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) mlx5_esw_for_each_rep(esw, i, rep) mlx5_esw_offloads_rep_cleanup(esw, rep); xa_destroy(&esw->offloads.vport_reps); + mutex_destroy(&esw->offloads.reps_lock); } static int esw_offloads_init_reps(struct mlx5_eswitch *esw) @@ -2654,6 +2677,7 @@ static int esw_offloads_init_reps(struct mlx5_eswitch *esw) int err; xa_init(&esw->offloads.vport_reps); + mutex_init(&esw->offloads.reps_lock); mlx5_esw_for_each_vport(esw, i, vport) { err = mlx5_esw_offloads_rep_add(esw, vport); @@ -2762,11 +2786,28 @@ void esw_offloads_cleanup(struct mlx5_eswitch *esw) } static int __esw_offloads_load_rep(struct mlx5_eswitch *esw, - struct mlx5_eswitch_rep *rep, u8 rep_type) + struct mlx5_eswitch_rep *rep, + u8 rep_type, bool *newly_loaded) { + int err; + + mlx5_esw_assert_reps_locked(esw); + + if (newly_loaded) + *newly_loaded = false; + if (atomic_cmpxchg(&rep->rep_data[rep_type].state, - REP_REGISTERED, REP_LOADED) == REP_REGISTERED) - return esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); + REP_REGISTERED, REP_LOADED) != REP_REGISTERED) + return 0; + + err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); + if (err) { + atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); + return err; + } + + if (newly_loaded) + *newly_loaded = true; return 0; } @@ -2774,6 +2815,8 @@ static int __esw_offloads_load_rep(struct mlx5_eswitch *esw, static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep, u8 rep_type) { + mlx5_esw_assert_reps_locked(esw); + if (atomic_cmpxchg(&rep->rep_data[rep_type].state, REP_LOADED, REP_REGISTERED) == REP_LOADED) { if (rep_type == REP_ETH) @@ -2794,22 +2837,27 @@ static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) { struct mlx5_eswitch_rep *rep; + unsigned long loaded = 0; + bool newly_loaded; int rep_type; int err; rep = mlx5_eswitch_get_rep(esw, vport_num); for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { - err = __esw_offloads_load_rep(esw, rep, rep_type); + err = __esw_offloads_load_rep(esw, rep, rep_type, + &newly_loaded); if (err) goto err_reps; + if (newly_loaded) + loaded |= BIT(rep_type); } return 0; err_reps: - atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); - for (--rep_type; rep_type >= 0; rep_type--) - __esw_offloads_unload_rep(esw, rep, rep_type); + while (--rep_type >= 0) + if (test_bit(rep_type, &loaded)) + __esw_offloads_unload_rep(esw, rep, rep_type); return err; } @@ -3563,13 +3611,13 @@ int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw) if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) return 0; - ret = __esw_offloads_load_rep(esw, rep, REP_IB); + ret = __esw_offloads_load_rep(esw, rep, REP_IB, NULL); if (ret) return ret; mlx5_esw_for_each_rep(esw, i, rep) { if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) - __esw_offloads_load_rep(esw, rep, REP_IB); + __esw_offloads_load_rep(esw, rep, REP_IB, NULL); } return 0; @@ -3679,6 +3727,7 @@ static void esw_vfs_changed_event_handler(struct mlx5_eswitch *esw) if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) goto free; + mlx5_esw_reps_block(esw); /* Number of VFs can only change from "0 to x" or "x to 0". */ if (esw->esw_funcs.num_vfs > 0) { mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); @@ -3688,9 +3737,11 @@ static void esw_vfs_changed_event_handler(struct mlx5_eswitch *esw) err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, MLX5_VPORT_UC_ADDR_CHANGE); if (err) - goto free; + goto unblock; } esw->esw_funcs.num_vfs = new_num_vfs; +unblock: + mlx5_esw_reps_unblock(esw); free: kvfree(out); } @@ -3736,11 +3787,12 @@ static void esw_wq_handler(struct work_struct *work) } static int mlx5_esw_add_work(struct mlx5_eswitch *esw, - void (*func)(struct mlx5_eswitch *esw)) + void (*func)(struct mlx5_eswitch *esw), + gfp_t gfp) { struct mlx5_host_work *host_work; - host_work = kzalloc_obj(*host_work, GFP_ATOMIC); + host_work = kzalloc_obj(*host_work, gfp); if (!host_work) return -ENOMEM; @@ -3764,7 +3816,8 @@ int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); - ret = mlx5_esw_add_work(esw, esw_vfs_changed_event_handler); + ret = mlx5_esw_add_work(esw, esw_vfs_changed_event_handler, + GFP_ATOMIC); if (ret) return NOTIFY_DONE; @@ -4176,9 +4229,14 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, goto unlock; } + /* Keep mode_lock and reps_lock unnested. The operation flag excludes + * mode users while mode_lock is dropped before taking reps_lock. + */ esw->eswitch_operation_in_progress = true; up_write(&esw->mode_lock); + mlx5_esw_reps_block(esw); + if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && !mlx5_devlink_netdev_netns_immutable_set(devlink, true)) { NL_SET_ERR_MSG_MOD(extack, @@ -4211,6 +4269,10 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, skip: if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && err) mlx5_devlink_netdev_netns_immutable_set(devlink, false); + /* Reconfiguration is done; drop reps_lock before taking mode_lock again + * to clear the operation flag. + */ + mlx5_esw_reps_unblock(esw); down_write(&esw->mode_lock); esw->eswitch_operation_in_progress = false; unlock: @@ -4484,9 +4546,10 @@ mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) return true; } -void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, - const struct mlx5_eswitch_rep_ops *ops, - u8 rep_type) +static void +mlx5_eswitch_register_vport_reps_blocked(struct mlx5_eswitch *esw, + const struct mlx5_eswitch_rep_ops *ops, + u8 rep_type) { struct mlx5_eswitch_rep_data *rep_data; struct mlx5_eswitch_rep *rep; @@ -4501,21 +4564,111 @@ void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, } } } + +static void mlx5_eswitch_reload_reps_blocked(struct mlx5_eswitch *esw) +{ + struct mlx5_vport *vport; + unsigned long i; + + if (esw->mode != MLX5_ESWITCH_OFFLOADS) + return; + + if (mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK)) + return; + + mlx5_esw_for_each_vport(esw, i, vport) { + if (!vport) + continue; + if (!vport->enabled) + continue; + if (vport->vport == MLX5_VPORT_UPLINK) + continue; + if (!mlx5_eswitch_vport_has_rep(esw, vport->vport)) + continue; + + mlx5_esw_offloads_rep_load(esw, vport->vport); + } +} + +static void mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw) +{ + mlx5_esw_reps_block(esw); + mlx5_eswitch_reload_reps_blocked(esw); + mlx5_esw_reps_unblock(esw); +} + +static void +mlx5_eswitch_register_vport_reps_locked(struct mlx5_eswitch *esw, + const struct mlx5_eswitch_rep_ops *ops, + u8 rep_type, bool nested) +{ + if (nested) + mlx5_esw_reps_block_nested(esw); + else + mlx5_esw_reps_block(esw); + mlx5_eswitch_register_vport_reps_blocked(esw, ops, rep_type); + mlx5_esw_reps_unblock(esw); + + mlx5_esw_add_work(esw, mlx5_eswitch_reload_reps, GFP_KERNEL); +} + +void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, + const struct mlx5_eswitch_rep_ops *ops, + u8 rep_type) +{ + mlx5_eswitch_register_vport_reps_locked(esw, ops, rep_type, false); +} EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); -void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) +void +mlx5_eswitch_register_vport_reps_nested(struct mlx5_eswitch *esw, + const struct mlx5_eswitch_rep_ops *ops, + u8 rep_type) +{ + mlx5_eswitch_register_vport_reps_locked(esw, ops, rep_type, true); +} +EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps_nested); + +static void +mlx5_eswitch_unregister_vport_reps_blocked(struct mlx5_eswitch *esw, + u8 rep_type) { struct mlx5_eswitch_rep *rep; unsigned long i; - if (esw->mode == MLX5_ESWITCH_OFFLOADS) - __unload_reps_all_vport(esw, rep_type); + __unload_reps_all_vport(esw, rep_type); mlx5_esw_for_each_rep(esw, i, rep) atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); + + esw->offloads.rep_ops[rep_type] = NULL; +} + +static void +mlx5_eswitch_unregister_vport_reps_locked(struct mlx5_eswitch *esw, + u8 rep_type, bool nested) +{ + if (nested) + mlx5_esw_reps_block_nested(esw); + else + mlx5_esw_reps_block(esw); + mlx5_eswitch_unregister_vport_reps_blocked(esw, rep_type); + mlx5_esw_reps_unblock(esw); +} + +void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) +{ + mlx5_eswitch_unregister_vport_reps_locked(esw, rep_type, false); } EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); +void mlx5_eswitch_unregister_vport_reps_nested(struct mlx5_eswitch *esw, + u8 rep_type) +{ + mlx5_eswitch_unregister_vport_reps_locked(esw, rep_type, true); +} +EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps_nested); + void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) { struct mlx5_eswitch_rep *rep; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 449e4bd86c06..e77f9931c39c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -1063,34 +1063,147 @@ bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) return true; } -void mlx5_lag_add_devices(struct mlx5_lag *ldev) +static void mlx5_lag_assert_locked_transition(struct mlx5_lag *ldev) { + struct mlx5_devcom_comp_dev *devcom = NULL; struct lag_func *pf; int i; + lockdep_assert_held(&ldev->lock); + + i = mlx5_get_next_ldev_func(ldev, 0); + if (i < MLX5_MAX_PORTS) { + pf = mlx5_lag_pf(ldev, i); + devcom = pf->dev->priv.hca_devcom_comp; + } + mlx5_devcom_comp_assert_locked(devcom); +} + +static void mlx5_lag_drop_lock_for_reps(struct mlx5_lag *ldev) +{ + mlx5_lag_assert_locked_transition(ldev); + + /* Keep PF membership stable while ldev->lock is dropped. Device add + * and remove paths observe mode_changes_in_progress and retry. + */ + ldev->mode_changes_in_progress++; + mutex_unlock(&ldev->lock); +} + +static void mlx5_lag_retake_lock_after_reps(struct mlx5_lag *ldev) +{ + mutex_lock(&ldev->lock); + ldev->mode_changes_in_progress--; +} + +void mlx5_lag_rescan_dev_locked(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev, + bool enable) +{ + if (dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) + return; + + if (enable) + dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + else + dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + + /* Auxiliary bus probe/remove can register or unregister representor + * callbacks and take reps_lock. Drop ldev->lock so the only ordering + * remains reps_lock -> ldev->lock from representor callbacks. + */ + mlx5_lag_drop_lock_for_reps(ldev); + mlx5_rescan_drivers_locked(dev); + mlx5_lag_retake_lock_after_reps(ldev); +} + +static void mlx5_lag_rescan_devices_locked(struct mlx5_lag *ldev, bool enable) +{ + struct mlx5_core_dev *devs[MLX5_MAX_PORTS]; + struct lag_func *pf; + int num_devs = 0; + int i; + + mlx5_lag_assert_locked_transition(ldev); + mlx5_ldev_for_each(i, 0, ldev) { pf = mlx5_lag_pf(ldev, i); if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) continue; - pf->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(pf->dev); + if (enable) + pf->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + else + pf->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + devs[num_devs++] = pf->dev; } + + mlx5_lag_drop_lock_for_reps(ldev); + for (i = 0; i < num_devs; i++) + mlx5_rescan_drivers_locked(devs[i]); + mlx5_lag_retake_lock_after_reps(ldev); +} + +void mlx5_lag_add_devices(struct mlx5_lag *ldev) +{ + mlx5_lag_rescan_devices_locked(ldev, true); } void mlx5_lag_remove_devices(struct mlx5_lag *ldev) +{ + mlx5_lag_rescan_devices_locked(ldev, false); +} + +static int mlx5_lag_reload_ib_reps_unlocked(struct mlx5_lag *ldev, u32 flags, + bool cont_on_fail) { struct lag_func *pf; + int ret; int i; mlx5_ldev_for_each(i, 0, ldev) { pf = mlx5_lag_pf(ldev, i); - if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) - continue; + if (!(pf->dev->priv.flags & flags)) { + struct mlx5_eswitch *esw; - pf->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(pf->dev); + esw = pf->dev->priv.eswitch; + mlx5_esw_reps_block(esw); + ret = mlx5_eswitch_reload_ib_reps(esw); + mlx5_esw_reps_unblock(esw); + if (ret && !cont_on_fail) + return ret; + } } + + return 0; +} + +static int mlx5_lag_reload_ib_reps(struct mlx5_lag *ldev, u32 flags, + bool cont_on_fail) +{ + int ret; + + /* The HCA devcom component lock serializes LAG mode transitions while + * ldev->lock is dropped here. Dropping ldev->lock is required because + * the reload takes the per-E-Switch reps_lock, and representor + * load/unload callbacks can re-enter LAG netdev add/remove and take + * ldev->lock. Keep the ordering reps_lock -> ldev->lock. + */ + mlx5_lag_drop_lock_for_reps(ldev); + ret = mlx5_lag_reload_ib_reps_unlocked(ldev, flags, cont_on_fail); + mlx5_lag_retake_lock_after_reps(ldev); + + return ret; +} + +int mlx5_lag_reload_ib_reps_from_locked(struct mlx5_lag *ldev, u32 flags, + bool cont_on_fail) +{ + int ret; + + ret = mlx5_lag_reload_ib_reps(ldev, flags, cont_on_fail); + + return ret; } void mlx5_disable_lag(struct mlx5_lag *ldev) @@ -1111,10 +1224,7 @@ void mlx5_disable_lag(struct mlx5_lag *ldev) if (shared_fdb) { mlx5_lag_remove_devices(ldev); } else if (roce_lag) { - if (!(dev0->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) { - dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - } + mlx5_lag_rescan_dev_locked(ldev, dev0, false); mlx5_ldev_for_each(i, 0, ldev) { if (i == idx) continue; @@ -1130,9 +1240,9 @@ void mlx5_disable_lag(struct mlx5_lag *ldev) mlx5_lag_add_devices(ldev); if (shared_fdb) - mlx5_ldev_for_each(i, 0, ldev) - if (!(mlx5_lag_pf(ldev, i)->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) - mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); + mlx5_lag_reload_ib_reps_from_locked(ldev, + MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV, + true); } bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev) @@ -1388,10 +1498,9 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) if (err) { if (shared_fdb || roce_lag) mlx5_lag_add_devices(ldev); - if (shared_fdb) { - mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - } + if (shared_fdb) + mlx5_lag_reload_ib_reps_from_locked(ldev, 0, + true); return; } @@ -1399,8 +1508,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) if (roce_lag) { struct mlx5_core_dev *dev; - dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); + mlx5_lag_rescan_dev_locked(ldev, dev0, true); mlx5_ldev_for_each(i, 0, ldev) { if (i == idx) continue; @@ -1409,24 +1517,15 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) mlx5_nic_vport_enable_roce(dev); } } else if (shared_fdb) { - int i; - - dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - - mlx5_ldev_for_each(i, 0, ldev) { - err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - if (err) - break; - } - + mlx5_lag_rescan_dev_locked(ldev, dev0, true); + err = mlx5_lag_reload_ib_reps_from_locked(ldev, 0, + false); if (err) { - dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); + mlx5_lag_rescan_dev_locked(ldev, dev0, false); mlx5_deactivate_lag(ldev); mlx5_lag_add_devices(ldev); - mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); + mlx5_lag_reload_ib_reps_from_locked(ldev, 0, + true); mlx5_core_err(dev0, "Failed to enable lag\n"); return; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h index 6c911374f409..6afe7707d076 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h @@ -164,6 +164,9 @@ void mlx5_disable_lag(struct mlx5_lag *ldev); void mlx5_lag_remove_devices(struct mlx5_lag *ldev); int mlx5_deactivate_lag(struct mlx5_lag *ldev); void mlx5_lag_add_devices(struct mlx5_lag *ldev); +void mlx5_lag_rescan_dev_locked(struct mlx5_lag *ldev, + struct mlx5_core_dev *dev, + bool enable); struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev); #ifdef CONFIG_MLX5_ESWITCH @@ -199,4 +202,6 @@ int mlx5_get_next_ldev_func(struct mlx5_lag *ldev, int start_idx); int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq); int mlx5_lag_num_devs(struct mlx5_lag *ldev); int mlx5_lag_num_netdevs(struct mlx5_lag *ldev); +int mlx5_lag_reload_ib_reps_from_locked(struct mlx5_lag *ldev, u32 flags, + bool cont_on_fail); #endif /* __MLX5_LAG_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c index 5eea12a6887a..8a349f8fd823 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c @@ -70,7 +70,6 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); struct mlx5_core_dev *dev0; int err; - int i; if (ldev->mode == MLX5_LAG_MODE_MPESW) return 0; @@ -101,26 +100,21 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) goto err_add_devices; } - dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); - mlx5_ldev_for_each(i, 0, ldev) { - err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); - if (err) - goto err_rescan_drivers; - } + mlx5_lag_rescan_dev_locked(ldev, dev0, true); + err = mlx5_lag_reload_ib_reps_from_locked(ldev, 0, false); + if (err) + goto err_rescan_drivers; mlx5_lag_set_vports_agg_speed(ldev); return 0; err_rescan_drivers: - dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(dev0); + mlx5_lag_rescan_dev_locked(ldev, dev0, false); mlx5_deactivate_lag(ldev); err_add_devices: mlx5_lag_add_devices(ldev); - mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); + mlx5_lag_reload_ib_reps_from_locked(ldev, 0, true); mlx5_mpesw_metadata_cleanup(ldev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c index 4b5ac2db55ce..d40c53193ea8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c @@ -3,6 +3,7 @@ #include #include +#include #include "lib/devcom.h" #include "lib/mlx5.h" #include "mlx5_core.h" @@ -438,3 +439,10 @@ int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom) return 0; return down_write_trylock(&devcom->comp->sem); } + +void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom) +{ + if (!devcom) + return; + lockdep_assert_held_write(&devcom->comp->sem); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index 91e5ae529d5c..316052a85ca5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -75,5 +75,6 @@ void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom); void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom); int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom); +void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom); #endif /* __LIB_MLX5_DEVCOM_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c index 8503e532f423..2fc69897e35b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -245,8 +245,10 @@ static int mlx5_sf_add(struct mlx5_core_dev *dev, struct mlx5_sf_table *table, if (IS_ERR(sf)) return PTR_ERR(sf); + mlx5_esw_reps_block(esw); err = mlx5_eswitch_load_sf_vport(esw, sf->hw_fn_id, MLX5_VPORT_UC_ADDR_CHANGE, &sf->dl_port, new_attr->controller, new_attr->sfnum); + mlx5_esw_reps_unblock(esw); if (err) goto esw_err; *dl_port = &sf->dl_port.dl_port; @@ -367,7 +369,10 @@ int mlx5_devlink_sf_port_del(struct devlink *devlink, struct mlx5_sf_table *table = dev->priv.sf_table; struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port); + mlx5_esw_reps_block(dev->priv.eswitch); mlx5_sf_del(table, sf); + mlx5_esw_reps_unblock(dev->priv.eswitch); + return 0; } diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index 3b29a3c6794d..a0dd162baa78 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -63,7 +63,13 @@ struct mlx5_eswitch_rep { void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, const struct mlx5_eswitch_rep_ops *ops, u8 rep_type); +void +mlx5_eswitch_register_vport_reps_nested(struct mlx5_eswitch *esw, + const struct mlx5_eswitch_rep_ops *ops, + u8 rep_type); void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type); +void mlx5_eswitch_unregister_vport_reps_nested(struct mlx5_eswitch *esw, + u8 rep_type); void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, u16 vport_num, u8 rep_type);