diff --git a/net/mac80211/link.c b/net/mac80211/link.c
index e81dd02de12e..d0535268962c 100644
--- a/net/mac80211/link.c
+++ b/net/mac80211/link.c
@@ -292,6 +292,7 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata,
 	u16 old_active = sdata->vif.active_links;
 	unsigned long add = new_links & ~old_links;
 	unsigned long rem = old_links & ~new_links;
+	unsigned long sta_rem = rem;
 	unsigned int link_id;
 	int ret;
 	struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS] = {}, *link;
@@ -299,6 +300,7 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata,
 	struct ieee80211_link_data *old_data[IEEE80211_MLD_MAX_NUM_LINKS];
 	bool use_deflink = old_links == 0; /* set for error case */
 	bool non_sta = sdata->vif.type != NL80211_IFTYPE_STATION;
+	struct sta_info *sta;
 
 	lockdep_assert_wiphy(sdata->local->hw.wiphy);
 
@@ -402,6 +404,34 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata,
 		goto free;
 	}
 
+	/* try to remove links that are now invalid from (MLO) stations */
+	list_for_each_entry(sta, &sdata->local->sta_list, list) {
+		unsigned long rem_links = sta->sta.valid_links & sta_rem;
+
+		if (sta->sdata != sdata)
+			continue;
+
+		/*
+		 * skip stations that would have no links left,
+		 * those will be removed completely later
+		 */
+		if (sta->sta.valid_links == rem_links)
+			continue;
+
+		for_each_set_bit(link_id, &rem_links,
+				 IEEE80211_MLD_MAX_NUM_LINKS)
+			ieee80211_sta_remove_link(sta, link_id);
+	}
+
+	/*
+	 * Remove stations using any removed links. Note that due
+	 * to the above station link removal, this only removes
+	 * stations that were skipped above because they'd have no
+	 * links left after link removal.
+	 */
+	for_each_set_bit(link_id, &sta_rem, IEEE80211_MLD_MAX_NUM_LINKS)
+		sta_info_flush(sdata, link_id);
+
 	/* use deflink/bss_conf again if and only if there are no more links */
 	use_deflink = new_links == 0;
 
diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
index cfb5bc4eac69..f55cb376cbd5 100644
--- a/net/mac80211/mlme.c
+++ b/net/mac80211/mlme.c
@@ -7169,7 +7169,6 @@ static void ieee80211_ml_reconf_work(struct wiphy *wiphy,
 		container_of(work, struct ieee80211_sub_if_data,
 			     u.mgd.ml_reconf_work.work);
 	u16 new_valid_links, new_active_links, new_dormant_links;
-	struct sta_info *sta;
 	int ret;
 
 	if (!sdata->u.mgd.removed_links)
@@ -7205,16 +7204,6 @@ static void ieee80211_ml_reconf_work(struct wiphy *wiphy,
 		}
 	}
 
-	sta = sta_info_get(sdata, sdata->vif.cfg.ap_addr);
-	if (sta) {
-		unsigned long removed_links = sdata->u.mgd.removed_links;
-		unsigned int link_id;
-
-		for_each_set_bit(link_id, &removed_links,
-				 IEEE80211_MLD_MAX_NUM_LINKS)
-			ieee80211_sta_remove_link(sta, link_id);
-	}
-
 	new_dormant_links = sdata->vif.dormant_links & ~sdata->u.mgd.removed_links;
 
 	ret = ieee80211_vif_set_links(sdata, new_valid_links,
@@ -11071,14 +11060,6 @@ int ieee80211_mgd_assoc_ml_reconf(struct ieee80211_sub_if_data *sdata,
 			goto err_free;
 		}
 
-		for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS;
-		     link_id++) {
-			if (!(req->rem_links & BIT(link_id)))
-				continue;
-
-			ieee80211_sta_remove_link(sta, link_id);
-		}
-
 		/* notify the driver and upper layers */
 		ieee80211_vif_cfg_change_notify(sdata,
 						BSS_CHANGED_MLD_VALID_LINKS);