diff --git a/net/core/filter.c b/net/core/filter.c index 09c60a58c82a..d832d3332c6f 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -8022,7 +8022,7 @@ BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags) if (sk_unhashed(sk)) return -EOPNOTSUPP; if (sk_is_refcounted(sk) && - unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) + unlikely(!refcount_inc_not_zero_acquire(&sk->sk_refcnt))) return -ENOENT; skb_orphan(skb); diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 93b306d91b6d..e56beec43877 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -1061,8 +1061,8 @@ static void reqsk_timer_handler(struct timer_list *t) * hold another count to prevent use-after-free and * call reqsk_put() just before return. */ - refcount_set(&nreq->rsk_refcnt, 2 + 1); timer_setup(&nreq->rsk_timer, reqsk_timer_handler, TIMER_PINNED); + refcount_set_release(&nreq->rsk_refcnt, 2 + 1); reqsk_queue_migrated(&inet_csk(nsk)->icsk_accept_queue, req); req = nreq; @@ -1141,10 +1141,14 @@ static void reqsk_timer_handler(struct timer_list *t) */ if (nreq) { __NET_INC_STATS(net, LINUX_MIB_TCPMIGRATEREQFAILURE); + /* The timer was not armed, drop its reference here. */ + reqsk_put(nreq); no_ownership: reqsk_migrate_reset(nreq); reqsk_queue_removed(queue, nreq); - __reqsk_free(nreq); + /* Drop the temporary reference and the one reserved for ehash. */ + reqsk_put(nreq); + reqsk_put(nreq); } drop: @@ -1443,7 +1447,7 @@ struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child, goto child_put; } - refcount_set(&nreq->rsk_refcnt, 1); + refcount_set_release(&nreq->rsk_refcnt, 1); if (inet_csk_reqsk_queue_add(sk, nreq, child)) { __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQSUCCESS); reqsk_migrate_reset(req); @@ -1453,7 +1457,7 @@ struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child, __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMIGRATEREQFAILURE); reqsk_migrate_reset(nreq); - __reqsk_free(nreq); + reqsk_put(nreq); } else if (inet_csk_reqsk_queue_add(sk, req, child)) { return child; } @@ -1496,7 +1500,7 @@ void inet_csk_listen_stop(struct sock *sk) if (nsk) { nreq = inet_reqsk_clone(req, nsk); if (nreq) { - refcount_set(&nreq->rsk_refcnt, 1); + refcount_set_release(&nreq->rsk_refcnt, 1); rcu_read_lock(); if (inet_csk_reqsk_queue_add(nsk, nreq, child)) { @@ -1508,7 +1512,7 @@ void inet_csk_listen_stop(struct sock *sk) __NET_INC_STATS(sock_net(nsk), LINUX_MIB_TCPMIGRATEREQFAILURE); reqsk_migrate_reset(nreq); - __reqsk_free(nreq); + reqsk_put(nreq); } rcu_read_unlock(); diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index 5abcb0debb27..860145d810ed 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -550,10 +550,11 @@ struct sock *__inet_lookup_established(const struct net *net, if (sk->sk_hash != hash) continue; if (likely(inet_match(net, sk, acookie, ports, dif, sdif))) { - if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) + if (unlikely(!refcount_inc_not_zero_acquire(&sk->sk_refcnt))) goto out; - if (unlikely(!inet_match(net, sk, acookie, - ports, dif, sdif))) { + if (unlikely(hlist_nulls_unhashed_lockless(&sk->sk_nulls_node) || + !inet_match(net, sk, acookie, ports, dif, + sdif))) { sock_gen_put(sk); goto begin; } diff --git a/net/ipv4/inet_timewait_sock.c b/net/ipv4/inet_timewait_sock.c index a1d86026aefb..e256c55ce9f2 100644 --- a/net/ipv4/inet_timewait_sock.c +++ b/net/ipv4/inet_timewait_sock.c @@ -143,7 +143,7 @@ void inet_twsk_hashdance_schedule(struct inet_timewait_sock *tw, * Also note that after this point, we lost our implicit reference * so we are not allowed to use tw anymore. */ - refcount_set(&tw->tw_refcnt, 3); + refcount_set_release(&tw->tw_refcnt, 3); /* Ensure tw_refcnt has been set before tw is published. * smp_wmb() provides the necessary memory barrier to enforce this @@ -330,7 +330,7 @@ void inet_twsk_purge(struct inet_hashinfo *hashinfo) if (check_net(sock_net(sk))) continue; - if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) + if (unlikely(!refcount_inc_not_zero_acquire(&sk->sk_refcnt))) continue; if (check_net(sock_net(sk))) { diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index 8c9691569e01..4e0ff3b82062 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -205,7 +205,7 @@ struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, child = icsk->icsk_af_ops->syn_recv_sock(sk, skb, req, dst, NULL, &own_req, NULL); if (child) { - refcount_set(&req->rsk_refcnt, 1); + refcount_set_release(&req->rsk_refcnt, 1); sock_rps_save_rxhash(child, skb); if (rsk_drop_req(req)) { @@ -218,6 +218,8 @@ struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb, bh_unlock_sock(child); sock_put(child); + reqsk_put(req); + return NULL; } __reqsk_free(req); diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c index 6dfe40322fb5..33451f00c4e8 100644 --- a/net/ipv4/tcp_fastopen.c +++ b/net/ipv4/tcp_fastopen.c @@ -366,7 +366,7 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk, tcp_reset_xmit_timer(child, ICSK_TIME_RETRANS, req->timeout, false); - refcount_set(&req->rsk_refcnt, 2); + refcount_set_release(&req->rsk_refcnt, 2); sk_mark_napi_id_set(child, skb); diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index ebd2c12e42d5..73c5b4189866 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -7825,7 +7825,9 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops, if (!inet_csk_reqsk_queue_add(sk, req, fastopen_sk)) { bh_unlock_sock(fastopen_sk); sock_put(fastopen_sk); - goto drop_and_free; + /* tcp_disconnect() already dropped one reqsk ref. */ + reqsk_put(req); + goto drop; } READ_ONCE(sk->sk_data_ready)(sk); bh_unlock_sock(fastopen_sk); diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index b111b51d69fc..1eea584d80df 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -105,10 +105,12 @@ struct sock *__inet6_lookup_established(const struct net *net, continue; if (!inet6_match(net, sk, saddr, daddr, ports, dif, sdif)) continue; - if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) + if (unlikely(!refcount_inc_not_zero_acquire(&sk->sk_refcnt))) goto out; - if (unlikely(!inet6_match(net, sk, saddr, daddr, ports, dif, sdif))) { + if (unlikely(hlist_nulls_unhashed_lockless(&sk->sk_nulls_node) || + !inet6_match(net, sk, saddr, daddr, ports, dif, + sdif))) { sock_gen_put(sk); goto begin; }