diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst index 4c6a35d07a08..208f46967ee5 100644 --- a/Documentation/networking/ip-sysctl.rst +++ b/Documentation/networking/ip-sysctl.rst @@ -2444,7 +2444,11 @@ fib_multipath_hash_policy - INTEGER Possible values: - - 0 - Layer 3 (source and destination addresses plus flow label) + - 0 - Layer 3 (source and destination addresses plus flow label). + For IPv6 TCP, the local ECMP path is selected from the socket + txhash rather than the flow label, and may change after a TCP + rehash event (such as a retransmission timeout) to recover from + path failure. The on-wire flow label is unaffected. - 1 - Layer 4 (standard 5-tuple) - 2 - Layer 3 or inner Layer 3 if present - 3 - Custom multipath hash. Fields used for multipath hash calculation diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 1dec81faff28..3de07e738538 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -955,6 +955,18 @@ static inline u32 ip6_multipath_hash_fields(const struct net *net) } #endif +/* Derive the IPv6 ECMP hash from txhash so a rehash may pick a different path; + * policy 0 only, and only when txhash is set. >> 1 clears the top bit + * (fib6_select_path() uses mp_hash as a signed 31-bit value); ?: 1 keeps the + * result non-zero, since mp_hash 0 falls back to rt6_multipath_hash(). + */ +static inline void ip6_ecmp_set_mp_hash(const struct net *net, + struct flowi6 *fl6, u32 txhash) +{ + if (ip6_multipath_hash_policy(net) == 0 && txhash) + fl6->mp_hash = (txhash >> 1) ?: 1; +} + /* * Header manipulation */ diff --git a/include/net/sock.h b/include/net/sock.h index 91ddc1a6fd60..51185222aac2 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2262,6 +2262,20 @@ sk_dst_reset(struct sock *sk) sk_dst_set(sk, NULL); } +/* Re-roll the socket txhash. On a rehash, IPv6 also drops the cached route + * so the next transmit re-selects an ECMP path; IPv4 keeps its route, since + * IPv4 ECMP path selection does not use sk_txhash. + */ +static inline bool __sk_rethink_txhash_reset_dst(struct sock *sk) +{ + if (sk_rethink_txhash(sk)) { + if (sk->sk_family == AF_INET6) + __sk_dst_reset(sk); + return true; + } + return false; +} + struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie); struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie); diff --git a/include/net/tcp.h b/include/net/tcp.h index f063eccbbba3..0632cf39d20a 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -2540,22 +2540,30 @@ extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops; #ifdef CONFIG_SYN_COOKIES static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, - const struct sock *sk, struct sk_buff *skb, - __u16 *mss) + struct sk_buff *skb, __u16 *mss) { - tcp_synq_overflow(sk); - __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); return ops->cookie_init_seq(skb, mss); } #else static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, - const struct sock *sk, struct sk_buff *skb, - __u16 *mss) + struct sk_buff *skb, __u16 *mss) { return 0; } #endif +#ifdef CONFIG_SYN_COOKIES +static inline void cookie_record_sent(const struct sock *sk) +{ + tcp_synq_overflow(sk); + __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); +} +#else +static inline void cookie_record_sent(const struct sock *sk) +{ +} +#endif + struct tcp_key { union { struct { diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index df479277fb80..73e129768184 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -280,9 +280,13 @@ static int cookie_tcp_reqsk_init(struct sock *sk, struct sk_buff *skb, treq->snt_synack = 0; treq->snt_tsval_first = 0; treq->tfo_listener = false; - treq->txhash = net_tx_rndhash(); treq->rcv_isn = ntohl(th->seq) - 1; treq->snt_isn = ntohl(th->ack_seq) - 1; + /* The request socket was freed after the SYN-ACK; use the cookie + * (snt_isn) as txhash so the full socket and the SYN-ACK make the + * same egress choice (IPv6 ECMP path; IPv4 TX queue). + */ + treq->txhash = treq->snt_isn; treq->syn_tos = TCP_SKB_CB(skb)->ip_dsfield; #if IS_ENABLED(CONFIG_MPTCP) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 8560a9c6d382..61045a8886e4 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -5022,8 +5022,9 @@ static void tcp_rcv_spurious_retrans(struct sock *sk, skb->protocol == htons(ETH_P_IPV6) && (tcp_sk(sk)->inet_conn.icsk_ack.lrcv_flowlabel != ntohl(ip6_flowlabel(ipv6_hdr(skb)))) && - sk_rethink_txhash(sk)) + __sk_rethink_txhash_reset_dst(sk)) { NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPDUPLICATEDATAREHASH); + } /* Save last flowlabel after a spurious retrans. */ tcp_save_lrcv_flowlabel(sk, skb); @@ -7635,6 +7636,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, tcp_rsk(req)->af_specific = af_ops; tcp_rsk(req)->ts_off = 0; tcp_rsk(req)->req_usec_ts = false; + tcp_rsk(req)->txhash = net_tx_rndhash(); #if IS_ENABLED(CONFIG_MPTCP) tcp_rsk(req)->is_mptcp = 0; #endif @@ -7658,7 +7660,15 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, /* Note: tcp_v6_init_req() might override ir_iif for link locals */ inet_rsk(req)->ir_iif = inet_request_bound_dev_if(sk, skb); - dst = af_ops->route_req(sk, skb, &fl, req, isn); + if (want_cookie) { + isn = cookie_init_sequence(af_ops, skb, &req->mss); + /* Use the cookie as txhash so the SYN-ACK and the later full + * socket make the same egress choice (IPv6 ECMP path; IPv4 TX queue). + */ + tcp_rsk(req)->txhash = isn; + } + + dst = af_ops->route_req(sk, skb, &fl, req, want_cookie ? 0 : isn); if (!dst) goto drop_and_free; @@ -7698,7 +7708,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, tcp_ecn_create_request(req, skb, sk, dst); if (want_cookie) { - isn = cookie_init_sequence(af_ops, sk, skb, &req->mss); + cookie_record_sent(sk); if (!tmp_opt.tstamp_ok) inet_rsk(req)->ecn_ok = 0; } @@ -7716,7 +7726,6 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, } #endif tcp_rsk(req)->snt_isn = isn; - tcp_rsk(req)->txhash = net_tx_rndhash(); tcp_rsk(req)->syn_tos = TCP_SKB_CB(skb)->ip_dsfield; tcp_openreq_init_rwin(req, sk, dst); sk_rx_queue_set(req_to_sk(req), skb); diff --git a/net/ipv4/tcp_plb.c b/net/ipv4/tcp_plb.c index c11a0cd3f8fe..bcc2f0add6af 100644 --- a/net/ipv4/tcp_plb.c +++ b/net/ipv4/tcp_plb.c @@ -78,7 +78,7 @@ void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb) if (plb->pause_until) return; - sk_rethink_txhash(sk); + __sk_rethink_txhash_reset_dst(sk); plb->consec_cong_rounds = 0; WRITE_ONCE(tcp_sk(sk)->plb_rehash, tcp_sk(sk)->plb_rehash + 1); NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPLBREHASH); diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 322db13333c7..bf171b5e1eb3 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -297,7 +297,7 @@ static int tcp_write_timeout(struct sock *sk) return 1; } - if (sk_rethink_txhash(sk)) { + if (__sk_rethink_txhash_reset_dst(sk)) { WRITE_ONCE(tp->timeout_rehash, tp->timeout_rehash + 1); __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPTIMEOUTREHASH); } diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 48dd7711d659..282912a11999 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -822,6 +822,8 @@ int inet6_sk_rebuild_header(struct sock *sk) fl6->flowi6_uid = sk_uid(sk); security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6)); + ip6_ecmp_set_mp_hash(sock_net(sk), fl6, sk->sk_txhash); + rcu_read_lock(); final_p = fl6_update_dst(fl6, rcu_dereference(np->opt), &np->final); rcu_read_unlock(); diff --git a/net/ipv6/inet6_connection_sock.c b/net/ipv6/inet6_connection_sock.c index 4665d84a7380..3e4ce8cb478e 100644 --- a/net/ipv6/inet6_connection_sock.c +++ b/net/ipv6/inet6_connection_sock.c @@ -48,6 +48,8 @@ struct dst_entry *inet6_csk_route_req(const struct sock *sk, fl6->flowi6_uid = sk_uid(sk); security_req_classify_flow(req, flowi6_to_flowi_common(fl6)); + ip6_ecmp_set_mp_hash(sock_net(sk), fl6, tcp_rsk(req)->txhash); + if (!dst) { dst = ip6_dst_lookup_flow(sock_net(sk), sk, fl6, final_p); if (IS_ERR(dst)) @@ -70,6 +72,9 @@ struct dst_entry *inet6_csk_route_socket(struct sock *sk, fl6->saddr = np->saddr; fl6->flowlabel = np->flow_label; IP6_ECN_flow_xmit(sk, fl6->flowlabel); + + if (sk->sk_protocol == IPPROTO_TCP) + ip6_ecmp_set_mp_hash(sock_net(sk), fl6, sk->sk_txhash); fl6->flowi6_oif = sk->sk_bound_dev_if; fl6->flowi6_mark = sk->sk_mark; fl6->fl6_sport = inet->inet_sport; diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index 4f6f0d751d6c..b581cb1ee2e8 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -245,6 +245,8 @@ struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb) fl6.flowi6_uid = sk_uid(sk); security_req_classify_flow(req, flowi6_to_flowi_common(&fl6)); + ip6_ecmp_set_mp_hash(net, &fl6, tcp_rsk(req)->txhash); + dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p); if (IS_ERR(dst)) { SKB_DR_SET(reason, IP_OUTNOROUTES); diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index c219f3334835..ebe161d72fbd 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -258,6 +258,8 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr, if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) saddr = &sk->sk_v6_rcv_saddr; + sk_set_txhash(sk); + fl6->flowi6_proto = IPPROTO_TCP; fl6->daddr = sk->sk_v6_daddr; fl6->saddr = saddr ? *saddr : np->saddr; @@ -275,6 +277,14 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr, security_sk_classify_flow(sk, flowi6_to_flowi_common(fl6)); + /* Non-zero mp_hash bypasses rt6_multipath_hash() in + * fib6_select_path(), letting txhash control ECMP path + * selection so that sk_rethink_txhash() rehashes onto a + * different path. Policies 1-3 derive a deterministic + * hash from the flow keys and must not be overridden. + */ + ip6_ecmp_set_mp_hash(net, fl6, sk->sk_txhash); + dst = ip6_dst_lookup_flow(net, sk, fl6, final_p); if (IS_ERR(dst)) { err = PTR_ERR(dst); @@ -315,8 +325,6 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr, if (err) goto late_failure; - sk_set_txhash(sk); - if (likely(!tp->repair)) { union tcp_seq_and_ts_off st; @@ -957,6 +965,13 @@ static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 if (txhash) { /* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */ skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4); + + /* Select the local ECMP path from the connection's txhash, + * so a control packet (RST, or ACK from a time-wait socket) + * uses the same nexthop as the data. Only policy 0 uses + * mp_hash; policies 1-3 derive a deterministic hash. + */ + ip6_ecmp_set_mp_hash(net, &fl6, txhash); } fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark; fl6.fl6_dport = t1->dest;