diff --git a/net/rds/connection.c b/net/rds/connection.c index 7c8ab8e973e1..b6c4beb50eaf 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -106,10 +106,12 @@ static struct rds_connection *rds_conn_lookup(struct net *net, } /* - * This is called by transports as they're bringing down a connection. - * It clears partial message state so that the transport can start sending - * and receiving over this connection again in the future. It is up to - * the transport to have serialized this call with its send and recv. + * This is called by rds_conn_shutdown() once the transport has brought + * a path down. It clears partial message state so that the transport + * can start sending and receiving over this path again in the future. + * The caller owns RDS_IN_XMIT and RDS_RECV_REFILL across this call, + * which is what serializes it against the send and receive-refill + * paths. */ static void rds_conn_path_reset(struct rds_conn_path *cp) { @@ -120,7 +122,16 @@ static void rds_conn_path_reset(struct rds_conn_path *cp) rds_stats_inc(s_conn_reset); rds_send_path_reset(cp); - cp->cp_flags = 0; + + /* Clear the bits the reset is responsible for individually: a + * blanket cp_flags = 0 is a plain store that can clobber a + * concurrent atomic read-modify-write on the same word. + * RDS_IN_XMIT and RDS_RECV_REFILL are held as locks by the + * caller, rds_conn_shutdown(), which releases them once the + * teardown is complete. + */ + clear_bit(RDS_LL_SEND_FULL, &cp->cp_flags); + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); /* Do not clear next_rx_seq here, else we cannot distinguish * retransmitted packets from new packets, and will hand all @@ -406,28 +417,70 @@ void rds_conn_shutdown(struct rds_conn_path *cp) } mutex_unlock(&cp->cp_cm_lock); + /* Quiesce the transmit and receive-refill paths by + * acquiring their bit locks, not merely waiting for + * them to be released: with a plain wait, either path + * can re-take its lock the instant after we sample it + * clear and then run concurrently with the transport + * shutdown and the path reset below. Holding both + * locks across the teardown makes that structurally + * impossible. + */ wait_event(cp->cp_waitq, - !test_bit(RDS_IN_XMIT, &cp->cp_flags)); + !test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags)); wait_event(cp->cp_waitq, - !test_bit(RDS_RECV_REFILL, &cp->cp_flags)); + !test_and_set_bit(RDS_RECV_REFILL, &cp->cp_flags)); conn->c_trans->conn_path_shutdown(cp); rds_conn_path_reset(cp); + /* Release the two locks and wake any waiter (e.g. + * rds_tcp_reset_callbacks()) that blocked on them while + * we held them. The unlock orders the transport's ring + * re-initialization and the path reset above before + * either bit is seen clear. rds_conn_path_reset() leaves + * both bits alone: ownership ends here, not inside the + * reset. + */ + clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags); + clear_bit_unlock(RDS_RECV_REFILL, &cp->cp_flags); + wake_up_all(&cp->cp_waitq); + if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING, - RDS_CONN_DOWN) && - !rds_conn_path_transition(cp, RDS_CONN_ERROR, RDS_CONN_DOWN)) { - /* This can happen - eg when we're in the middle of tearing - * down the connection, and someone unloads the rds module. - * Quite reproducible with loopback connections. - * Mostly harmless. + /* The path was dropped again while we tore it + * down: by a socket state-change callback in + * irq context on receipt of a FIN, or by an + * accept that claimed the path just before a + * drop put it back to RDS_CONN_ERROR and then + * installed a fresh socket on it. Unless a + * pending destroy suppressed it, the drop also + * queued another shutdown pass, and that pass + * must run, because it is what tears down + * whatever attached to the path after the + * transport shutdown above sampled its state. + * Consuming the RDS_CONN_ERROR here would turn + * that pass into a no-op: leave the state + * alone, and let the pass finish the job. * - * Note that this also happens with rds-tcp because - * we could have triggered rds_conn_path_drop in irq - * mode from rds_tcp_state change on the receipt of - * a FIN, thus we need to recheck for RDS_CONN_ERROR - * here. + * Quiesce the reconnect timer before bailing + * out, though. When a pending destroy did + * suppress the queue, no later pass runs, and + * rds_conn_path_destroy() is about to flush + * cp_down_w and free the path: it must not + * find cp_conn_w still armed. A successor + * pass, when there is one, re-arms the + * reconnect from its own tail. + */ + cancel_delayed_work_sync(&cp->cp_conn_w); + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); + + if (rds_conn_path_state(cp) == RDS_CONN_ERROR) + return; + /* No current cp_state writer leaves a + * DISCONNECTING path in any state but + * RDS_CONN_ERROR; report loudly if one ever + * does. */ rds_conn_path_error(cp, "%s: failed to transition " "to state DOWN, current state " diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c index 357128d34a54..bd6cb3ffaa57 100644 --- a/net/rds/ib_recv.c +++ b/net/rds/ib_recv.c @@ -363,15 +363,14 @@ static int acquire_refill(struct rds_connection *conn) static void release_refill(struct rds_connection *conn) { - clear_bit(RDS_RECV_REFILL, &conn->c_flags); - smp_mb__after_atomic(); + clear_bit_unlock(RDS_RECV_REFILL, &conn->c_flags); /* We don't use wait_on_bit()/wake_up_bit() because our waking is in a * hot path and finding waiters is very rare. We don't want to walk * the system-wide hashed waitqueue buckets in the fast path only to * almost never find waiters. */ - if (waitqueue_active(&conn->c_waitq)) + if (wq_has_sleeper(&conn->c_waitq)) wake_up_all(&conn->c_waitq); } @@ -392,7 +391,9 @@ void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp) /* the goal here is to just make sure that someone, somewhere * is posting buffers. If we can't get the refill lock, - * let them do their thing + * let them do their thing. The holder may also be + * rds_conn_shutdown() tearing the path down, in which case + * there is nothing to post. */ if (!acquire_refill(conn)) return; diff --git a/net/rds/send.c b/net/rds/send.c index 15a1b97f13e7..1afa981e5c06 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -114,8 +114,13 @@ static void release_in_xmit(struct rds_conn_path *cp) * hot path and finding waiters is very rare. We don't want to walk * the system-wide hashed waitqueue buckets in the fast path only to * almost never find waiters. + * + * wq_has_sleeper() supplies the full barrier that orders the wait + * queue read after the bit clear; clear_bit_unlock() alone is only + * a release and would let this check read a stale empty queue, + * losing the wake-up. */ - if (waitqueue_active(&cp->cp_waitq)) + if (wq_has_sleeper(&cp->cp_waitq)) wake_up_all(&cp->cp_waitq); } @@ -239,8 +244,11 @@ int rds_send_xmit(struct rds_conn_path *cp) WRITE_ONCE(cp->cp_send_gen, send_gen); /* - * rds_conn_shutdown() sets the conn state and then tests RDS_IN_XMIT, - * we do the opposite to avoid races. + * rds_conn_shutdown() sets the conn state and then acquires + * RDS_IN_XMIT; we take the lock first and then check the state. + * Ownership is decided by the atomic RMW on the cp_flags word: + * if the teardown won the bit we back off here, and if we won + * it the teardown waits until we release it. */ if (!rds_conn_path_up(cp)) { release_in_xmit(cp); diff --git a/net/rds/tcp.c b/net/rds/tcp.c index b263634ac750..774a71f88d37 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -115,46 +115,90 @@ void rds_tcp_restore_callbacks(struct socket *sock, } /* - * rds_tcp_reset_callbacks() switches the to the new sock and - * returns the existing tc->t_sock. + * rds_tcp_reset_callbacks() switches a path to a new socket and + * releases the old one it finds in tc->t_sock, resolving a duelling + * SYN. * - * The only functions that set tc->t_sock are rds_tcp_set_callbacks - * and rds_tcp_reset_callbacks. Send and receive trust that - * it is set. The absence of RDS_CONN_UP bit protects those paths - * from being called while it isn't set. + * tc->t_sock is set by rds_tcp_set_callbacks() and cleared by + * rds_tcp_restore_callbacks(). Four paths write it: the active + * connect in rds_tcp_conn_path_connect(), which sets it and clears it + * again on failure; the accept path in rds_tcp_accept_one(), which + * sets it for a path with no socket yet; the teardown in + * rds_tcp_conn_path_shutdown(), which clears it; and the swap done + * here, which does both. The connect and accept paths are serialized + * against each other by t_conn_path_lock. Send and receive trust + * that it is set: the absence of RDS_CONN_UP protects those paths + * from being called while it isn't, and the swap done here runs under + * RDS_IN_XMIT so that it cannot interleave with a sender already + * inside rds_send_xmit(). */ void rds_tcp_reset_callbacks(struct socket *sock, struct rds_conn_path *cp) { struct rds_tcp_connection *tc = cp->cp_transport_data; - struct socket *osock = tc->t_sock; - - if (!osock) - goto newsock; + struct socket *osock; /* Need to resolve a duelling SYN between peers. * We have an outstanding SYN to this peer, which may * potentially have transitioned to the RDS_CONN_UP state, * so we must quiesce any send threads before resetting - * cp_transport_data. We quiesce these threads by setting - * cp_state to something other than RDS_CONN_UP, and then - * waiting for any existing threads in rds_send_xmit to - * complete release_in_xmit(). (Subsequent threads entering - * rds_send_xmit() will bail on !rds_conn_up(). + * cp_transport_data. Setting cp_state to something other + * than RDS_CONN_UP stops new senders, and owning RDS_IN_XMIT + * excludes any thread already inside rds_send_xmit() - or a + * teardown in rds_conn_shutdown(), which holds the same lock + * for the duration of the transport shutdown - for the whole + * socket swap and the rds_send_path_reset() below. * - * However an incoming syn-ack at this point would end up - * marking the conn as RDS_CONN_UP, and would again permit - * rds_send_xmi() threads through, so ideally we would - * synchronize on RDS_CONN_UP after lock_sock(), but cannot - * do that: waiting on !RDS_IN_XMIT after lock_sock() may - * end up deadlocking with tcp_sendmsg(), and the RDS_IN_XMIT - * would not get set. As a result, we set c_state to - * RDS_CONN_RESETTTING, to ensure that rds_tcp_state_change - * cannot mark rds_conn_path_up() in the window before lock_sock() + * An incoming syn-ack at this point would end up marking the + * conn as RDS_CONN_UP, and would again permit rds_send_xmit() + * threads through, so ideally we would synchronize on + * RDS_CONN_UP after lock_sock(), but cannot do that: acquiring + * RDS_IN_XMIT after lock_sock() may end up deadlocking with + * tcp_sendmsg(), which takes the socket lock while holding + * RDS_IN_XMIT. As a result, we set c_state to + * RDS_CONN_RESETTING, to ensure that rds_tcp_state_change + * cannot mark rds_conn_path_up() in the window before + * lock_sock(). + * + * Only make that transition if the path is still connecting + * (or already resetting from an earlier duel). A path in any + * other state - typically RDS_CONN_DISCONNECTING or + * RDS_CONN_ERROR with a shutdown in flight - is dropped + * instead. That still replaces its state, with RDS_CONN_ERROR, + * and, unless a pending destroy is about to reap the whole + * connection anyway, queues one more shutdown pass. A shutdown + * already in flight leaves that RDS_CONN_ERROR alone when it + * finishes; the queued pass then completes the transition to + * RDS_CONN_DOWN and tears down anything that attached to the + * path in the meantime. + */ + if (!rds_conn_path_transition(cp, RDS_CONN_CONNECTING, + RDS_CONN_RESETTING) && + !rds_conn_path_transition(cp, RDS_CONN_RESETTING, + RDS_CONN_RESETTING)) + rds_conn_path_drop(cp, 0); + wait_event(cp->cp_waitq, + !test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags)); + + /* Read t_sock only while owning RDS_IN_XMIT, never before the + * wait: the teardown in rds_conn_shutdown() releases the old + * socket and clears t_sock, so a pointer sampled earlier can + * be stale by the time we wake up. The teardown holds the + * same lock while it does so, so what we read here cannot + * change under us until we release it. + */ + osock = tc->t_sock; + if (!osock) + goto newsock; + + /* reset receive side state for rds_tcp_data_recv() for osock. + * + * The sync cancels while owning RDS_IN_XMIT rely on cp_wq + * being ordered: a teardown blocked on the bit occupies + * cp_wq's only execution slot, so cp_send_w and cp_recv_w are + * pending at most and the cancels never flush. Nothing here + * may flush or wait on cp_wq itself. */ - atomic_set(&cp->cp_state, RDS_CONN_RESETTING); - wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags)); - /* reset receive side state for rds_tcp_data_recv() for osock */ cancel_delayed_work_sync(&cp->cp_send_w); cancel_delayed_work_sync(&cp->cp_recv_w); lock_sock(osock->sk); @@ -172,6 +216,9 @@ void rds_tcp_reset_callbacks(struct socket *sock, lock_sock(sock->sk); rds_tcp_set_callbacks(sock, cp); release_sock(sock->sk); + + clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags); + wake_up_all(&cp->cp_waitq); } /* Add tc to rds_tcp_tc_list and set tc->t_sock. See comments diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c index a3db9b057084..13fa60c1985b 100644 --- a/net/rds/tcp_listen.c +++ b/net/rds/tcp_listen.c @@ -295,7 +295,11 @@ int rds_tcp_accept_one(struct rds_tcp_net *rtn) if (rs_tcp->t_sock) { /* Duelling SYN has been handled in rds_tcp_accept_one() */ rds_tcp_reset_callbacks(new_sock, cp); - /* rds_connect_path_complete() marks RDS_CONN_UP */ + /* rds_connect_path_complete() marks RDS_CONN_UP, or, + * if a concurrent shutdown won the duel, drops the + * path again and the pass that drop queues reaps the + * socket installed above. + */ rds_connect_path_complete(cp, RDS_CONN_RESETTING); } else { rds_tcp_set_callbacks(new_sock, cp);