mirror of
https://github.com/torvalds/linux.git
synced 2026-07-28 10:09:10 +02:00
Merge branch 'tcp-move-some-fastpath-fields-to-appropriate-groups'
Eric Dumazet says: ==================== tcp: move some fastpath fields to appropriate groups Move following fields to better groups to increase data locality. - delivered - delivered_ce - segs_in - segs_out - first_tx_mstamp - delivered_mstamp - max_packets_out - cwnd_usage_seq - rate_delivered - rate_interval_us No change in overall tcp_sock size. ==================== Link: https://patch.msgid.link/20260430100021.211139-1-edumazet@google.com Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski
commit
cd8886e981
|
|
@ -13,16 +13,16 @@ u16 tcp_header_len read_mostly read_m
|
|||
u16 gso_segs read_mostly tcp_xmit_size_goal
|
||||
__be32 pred_flags read_write read_mostly tcp_select_window(tx);tcp_rcv_established(rx)
|
||||
u64 bytes_received read_write tcp_rcv_nxt_update(rx)
|
||||
u32 segs_in read_write tcp_v6_rcv(rx)
|
||||
u32 segs_in read_write read_write tcp_segs_in(),tcp_v6_rcv(rx),tcp_v4_rcv()
|
||||
u32 data_segs_in read_write tcp_v6_rcv(rx)
|
||||
u32 rcv_nxt read_mostly read_write tcp_cleanup_rbuf,tcp_send_ack,tcp_inq_hint,tcp_transmit_skb,tcp_receive_window(tx);tcp_v6_do_rcv,tcp_rcv_established,tcp_data_queue,tcp_receive_window,tcp_rcv_nxt_update(write)(rx)
|
||||
u32 copied_seq read_mostly tcp_cleanup_rbuf,tcp_rcv_space_adjust,tcp_inq_hint
|
||||
u32 rcv_wup read_write __tcp_cleanup_rbuf,tcp_receive_window,tcp_receive_established
|
||||
u32 snd_nxt read_write read_mostly tcp_rate_check_app_limited,__tcp_transmit_skb,tcp_event_new_data_sent(write)(tx);tcp_rcv_established,tcp_ack,tcp_clean_rtx_queue(rx)
|
||||
u32 segs_out read_write __tcp_transmit_skb
|
||||
u32 segs_out read_write read_write __tcp_transmit_skb
|
||||
u32 data_segs_out read_write __tcp_transmit_skb,tcp_update_skb_after_send
|
||||
u64 bytes_sent read_write __tcp_transmit_skb
|
||||
u64 bytes_acked read_write tcp_snd_una_update/tcp_ack
|
||||
u64 bytes_acked read_write tcp_snd_una_update/tcp_ack
|
||||
u32 dsack_dups
|
||||
u32 snd_una read_mostly read_write tcp_wnd_end,tcp_urg_mode,tcp_minshall_check,tcp_cwnd_validate(tx);tcp_ack,tcp_may_update_window,tcp_clean_rtx_queue(write),tcp_ack_tstamp(rx)
|
||||
u32 snd_sml read_write tcp_minshall_check,tcp_minshall_update
|
||||
|
|
@ -99,8 +99,8 @@ u32 snd_cwnd_stamp
|
|||
u32 prior_cwnd
|
||||
u32 prr_delivered
|
||||
u32 prr_out read_mostly read_mostly tcp_rate_skb_sent,tcp_newly_delivered(tx);tcp_ack,tcp_rate_gen,tcp_clean_rtx_queue(rx)
|
||||
u32 delivered read_mostly read_write tcp_rate_skb_sent, tcp_newly_delivered(tx);tcp_ack, tcp_rate_gen, tcp_clean_rtx_queue (rx)
|
||||
u32 delivered_ce read_mostly read_write tcp_rate_skb_sent(tx);tcp_rate_gen(rx)
|
||||
u32 delivered read_write tcp_rate_skb_sent, tcp_newly_delivered(tx);tcp_ack, tcp_rate_gen, tcp_clean_rtx_queue (rx)
|
||||
u32 delivered_ce read_write tcp_rate_skb_sent(tx);tcp_rate_gen(rx)
|
||||
u32 received_ce read_mostly read_write
|
||||
u32[3] received_ecn_bytes read_mostly read_write
|
||||
u8:4 received_ce_pending read_mostly read_write
|
||||
|
|
|
|||
|
|
@ -253,15 +253,20 @@ struct tcp_sock {
|
|||
|
||||
/* TX read-write hotpath cache lines */
|
||||
__cacheline_group_begin(tcp_sock_write_tx) ____cacheline_aligned;
|
||||
u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut
|
||||
* The total number of segments sent.
|
||||
*/
|
||||
u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut
|
||||
* total number of data segments sent.
|
||||
u32 delivered; /* Total data packets delivered incl. rexmits */
|
||||
u32 delivered_ce; /* Like the above but only ECE marked packets */
|
||||
u64 bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked
|
||||
* sum(delta(snd_una)), or how many bytes
|
||||
* were acked.
|
||||
*/
|
||||
u64 bytes_sent; /* RFC4898 tcpEStatsPerfHCDataOctetsOut
|
||||
* total number of data bytes sent.
|
||||
*/
|
||||
u64 first_tx_mstamp; /* start of window send phase */
|
||||
u64 delivered_mstamp; /* time we reached "delivered" */
|
||||
u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut
|
||||
* total number of data segments sent.
|
||||
*/
|
||||
u32 snd_sml; /* Last byte of the most recently transmitted small packet */
|
||||
u8 chrono_type; /* current chronograph type */
|
||||
u32 chrono_start; /* Start time in jiffies of a TCP chrono */
|
||||
|
|
@ -271,6 +276,10 @@ struct tcp_sock {
|
|||
u32 lsndtime; /* timestamp of last sent data packet (for restart window) */
|
||||
u32 mdev_us; /* medium deviation */
|
||||
u32 rtt_seq; /* sequence number to update rttvar */
|
||||
u32 max_packets_out; /* max packets_out in last window */
|
||||
u32 cwnd_usage_seq; /* right edge of cwnd usage tracking flight */
|
||||
u32 rate_delivered; /* saved rate sample: packets delivered */
|
||||
u32 rate_interval_us; /* saved rate sample: time elapsed */
|
||||
u64 tcp_wstamp_ns; /* departure time for next sent data packet */
|
||||
u64 accecn_opt_tstamp; /* Last AccECN option sent timestamp */
|
||||
struct list_head tsorted_sent_queue; /* time-sorted sent but un-SACKed skbs */
|
||||
|
|
@ -307,8 +316,6 @@ struct tcp_sock {
|
|||
u32 srtt_us; /* smoothed round trip time << 3 in usecs */
|
||||
u32 packets_out; /* Packets which are "in flight" */
|
||||
u32 snd_up; /* Urgent pointer */
|
||||
u32 delivered; /* Total data packets delivered incl. rexmits */
|
||||
u32 delivered_ce; /* Like the above but only ECE marked packets */
|
||||
u32 received_ce; /* Like the above but for rcvd CE marked pkts */
|
||||
u32 received_ecn_bytes[3]; /* received byte counters for three ECN
|
||||
* types: INET_ECN_ECT_1, INET_ECN_ECT_0,
|
||||
|
|
@ -324,6 +331,12 @@ struct tcp_sock {
|
|||
* Options received (usually on last packet, some only on SYN packets).
|
||||
*/
|
||||
struct tcp_options_received rx_opt;
|
||||
u32 segs_in; /* RFC4898 tcpEStatsPerfSegsIn
|
||||
* total number of segments in.
|
||||
*/
|
||||
u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut
|
||||
* The total number of segments sent.
|
||||
*/
|
||||
__cacheline_group_end(tcp_sock_write_txrx);
|
||||
|
||||
/* RX read-write hotpath cache lines */
|
||||
|
|
@ -333,26 +346,13 @@ struct tcp_sock {
|
|||
* sum(delta(rcv_nxt)), or how many bytes
|
||||
* were acked.
|
||||
*/
|
||||
u32 segs_in; /* RFC4898 tcpEStatsPerfSegsIn
|
||||
* total number of segments in.
|
||||
*/
|
||||
u32 data_segs_in; /* RFC4898 tcpEStatsPerfDataSegsIn
|
||||
* total number of data segments in.
|
||||
*/
|
||||
u32 rcv_wup; /* rcv_nxt on last window update sent */
|
||||
u32 max_packets_out; /* max packets_out in last window */
|
||||
u32 cwnd_usage_seq; /* right edge of cwnd usage tracking flight */
|
||||
u32 rate_delivered; /* saved rate sample: packets delivered */
|
||||
u32 rate_interval_us; /* saved rate sample: time elapsed */
|
||||
u32 rcv_rtt_last_tsecr;
|
||||
u32 delivered_ecn_bytes[3];
|
||||
u16 pkts_acked_ewma;/* Pkts acked EWMA for AccECN cep heuristic */
|
||||
u64 first_tx_mstamp; /* start of window send phase */
|
||||
u64 delivered_mstamp; /* time we reached "delivered" */
|
||||
u64 bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked
|
||||
* sum(delta(snd_una)), or how many bytes
|
||||
* were acked.
|
||||
*/
|
||||
struct {
|
||||
u32 rtt_us;
|
||||
u32 seq;
|
||||
|
|
|
|||
|
|
@ -5230,9 +5230,13 @@ static void __init tcp_struct_check(void)
|
|||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh);
|
||||
|
||||
/* TX read-write hotpath cache lines */
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered_ce);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_acked);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, first_tx_mstamp);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered_mstamp);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat);
|
||||
|
|
@ -5243,6 +5247,10 @@ static void __init tcp_struct_check(void)
|
|||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, accecn_opt_tstamp);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, max_packets_out);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, cwnd_usage_seq);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rate_delivered);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rate_interval_us);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags);
|
||||
|
|
@ -5258,8 +5266,6 @@ static void __init tcp_struct_check(void)
|
|||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ce);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ecn_bytes);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited);
|
||||
|
|
@ -5267,22 +5273,16 @@ static void __init tcp_struct_check(void)
|
|||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_mwnd_seq);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_tstamp);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, segs_in);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, segs_out);
|
||||
|
||||
/* RX read-write hotpath cache lines */
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_ecn_bytes);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, pkts_acked_ewma);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est);
|
||||
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user