diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 88167edc6598..09cd3c1f3cdb 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -1142,13 +1142,6 @@ static void __mptcp_clean_una_wakeup(struct sock *sk) mptcp_write_space(sk); } -static void mptcp_clean_una_wakeup(struct sock *sk) -{ - mptcp_data_lock(sk); - __mptcp_clean_una_wakeup(sk); - mptcp_data_unlock(sk); -} - static void mptcp_enter_memory_pressure(struct sock *sk) { struct mptcp_subflow_context *subflow; @@ -2790,8 +2783,12 @@ static void mptcp_check_fastclose(struct mptcp_sock *msk) sk_error_report(sk); } -/* Retransmit the specified data fragment on all the selected subflows. */ -static int __mptcp_push_retrans(struct sock *sk, struct mptcp_data_frag *dfrag) +/* + * Retransmit the specified data fragment on all the selected subflows, + * starting from the specified sequence + */ +static int __mptcp_push_retrans(struct sock *sk, struct mptcp_data_frag *dfrag, + u64 sent_seq) { struct mptcp_sendmsg_info info = { .data_lock_held = true, }; struct mptcp_sock *msk = mptcp_sk(sk); @@ -2801,6 +2798,7 @@ static int __mptcp_push_retrans(struct sock *sk, struct mptcp_data_frag *dfrag) mptcp_for_each_subflow(msk, subflow) { if (READ_ONCE(subflow->scheduled)) { + u16 offset = sent_seq - dfrag->data_seq; u16 copied = 0; mptcp_subflow_set_scheduled(subflow, false); @@ -2810,7 +2808,7 @@ static int __mptcp_push_retrans(struct sock *sk, struct mptcp_data_frag *dfrag) lock_sock(ssk); /* limit retransmission to the bytes already sent on some subflows */ - info.sent = 0; + info.sent = offset; info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent; @@ -2856,15 +2854,78 @@ static void __mptcp_retrans(struct sock *sk) struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_subflow_context *subflow; struct mptcp_data_frag *dfrag; + u64 retrans_seq, sent_seq; + bool need_retrans; int err, len; mptcp_pm_chk_stale(msk); - mptcp_clean_una_wakeup(sk); - - err = mptcp_sched_get_retrans(msk); + /* Get an updated and consistent rtx queue status. */ + mptcp_data_lock(sk); + __mptcp_clean_una_wakeup(sk); + retrans_seq = msk->snd_una; dfrag = mptcp_rtx_head(sk); - if (!dfrag) { + need_retrans = !!dfrag; + mptcp_data_unlock(sk); + + for (;;) { + bool already_acked; + + err = mptcp_sched_get_retrans(msk); + if (err) + break; + + /* `already_sent` can be 0 for `dfrag` belonging to the RTX + * queue due to __mptcp_retransmit_pending_data(). + */ + if (!dfrag || !dfrag->already_sent) + break; + + /* Can fail only in case of fallback. */ + len = __mptcp_push_retrans(sk, dfrag, retrans_seq); + if (len < 0) + goto clear_scheduled; + + retrans_seq += len; + msk->bytes_retrans += len; + dfrag->already_sent = max_t(u16, dfrag->already_sent, + retrans_seq - dfrag->data_seq); + + /* With csum enabled, retransmission can send new data. */ + sent_seq = dfrag->already_sent + dfrag->data_seq; + if (after64(sent_seq, msk->snd_nxt)) + WRITE_ONCE(msk->snd_nxt, sent_seq); + + /* Attempt the next fragment only if the current one is + * completely retransmitted. + */ + if (before64(retrans_seq, dfrag->data_seq + dfrag->data_len)) + break; + + dfrag = list_is_last(&dfrag->list, &msk->rtx_queue) ? + NULL : list_next_entry(dfrag, list); + if (!dfrag) + break; + + /* Incoming acks can move snd_una after the current dfrag + * across loop iterations, if so start again from RTX head. + */ + mptcp_data_lock(sk); + already_acked = !before64(msk->snd_una, dfrag->data_seq + + dfrag->already_sent); + if (already_acked) { + __mptcp_clean_una_wakeup(sk); + retrans_seq = msk->snd_una; + dfrag = mptcp_rtx_head(sk); + need_retrans = !!dfrag; + } else if (after64(msk->snd_una, retrans_seq)) { + retrans_seq = msk->snd_una; + } + mptcp_data_unlock(sk); + } + + /* Attempt data-fin retransmission only when the RTX queue is empty. */ + if (!need_retrans) { if (mptcp_data_fin_enabled(msk)) { struct inet_connection_sock *icsk = inet_csk(sk); @@ -2872,30 +2933,13 @@ static void __mptcp_retrans(struct sock *sk) icsk->icsk_retransmits + 1); mptcp_set_datafin_timeout(sk); mptcp_send_ack(msk); - goto reset_timer; } if (!mptcp_send_head(sk)) goto clear_scheduled; - - goto reset_timer; } - if (err) - goto reset_timer; - - len = __mptcp_push_retrans(sk, dfrag); - if (len < 0) - goto clear_scheduled; - - msk->bytes_retrans += len; - dfrag->already_sent = max(dfrag->already_sent, len); - - /* With csum enabled retransmission can send new data. */ - if (after64(dfrag->already_sent + dfrag->data_seq, msk->snd_nxt)) - WRITE_ONCE(msk->snd_nxt, dfrag->already_sent + dfrag->data_seq); - reset_timer: mptcp_check_and_set_pending(sk);