github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: tcp: Check BPF_SOCK_OPS_TEST_FLAG() after cgroup_bpf_enabled(CGROUP_SOCK_OPS).

Browse Source 
BPF_CGROUP_RUN_PROG_XXX() macros guard __cgroup_bpf_run_filter_XXX()
with cgroup_bpf_enabled().

However, even when no SOCK_OPS prog is attached, callers still
initialise struct bpf_sock_ops_kern (memset(), etc.) or evaluate
BPF_SOCK_OPS_TEST_FLAG(), which loads tp->bpf_sock_ops_cb_flags
from a cold cacheline near the end of struct tcp_sock.

Similar to bpf_tcp_ops, let's check cgroup_bpf_enabled() before
BPF_SOCK_OPS_TEST_FLAG() and struct bpf_sock_ops_kern setup, and
rename BPF_CGROUP_RUN_PROG_SOCK_OPS{,_SK}() with __ prefix.

Since all direct callers of tcp_call_bpf() pass 0 and NULL for
nargs and args, they can be folded into the new tcp_call_bpf()
macro.

All callers of tcp_call_bpf_{2,3}arg() check BPF_SOCK_OPS_TEST_FLAG()
and do not need the return value.  These are replaced with the new
tcp_call_bpf_flag() macro.

Now, bpf_skops_hdr_opt_len() and bpf_tcp_ops_hdr_opt_len() are
just NOPs without bpf prog attached.

  $ llvm-objdump -S -D --disassemble=tcp_established_options vmlinux
  ...
  ; 	asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]")
  ; 	size = MAX_TCP_OPTION_SPACE - remaining;
  ffffffff825127ea: b9 28 00 00 00       	movl	$0x28, %ecx
  ffffffff825127ef: 29 c1                	subl	%eax, %ecx
  ; 	return size;
  ffffffff825127f1: 89 c8                	movl	%ecx, %eax
  ...
  ffffffff825127fb: 2e e9 6f 02 43 00    	jmp	0xffffffff82942a70 <__x86_return_thunk>

Signed-off-by: Kuniyuki Iwashima <kuniyu@google.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Reviewed-by: Amery Hung <ameryhung@gmail.com>
Acked-by: Stanislav Fomichev <sdf@fomichev.me>
Link: https://patch.msgid.link/20261008031604.256498-6-kuniyu@google.com
This commit is contained in:
Kuniyuki Iwashima 2026-10-08 03:15:25 +00:00 committed by Alexei Starovoitov
parent 59c25bb675
commit c071408db3
No known key found for this signature in database
8 changed files with 69 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
include/linux/bpf-cgroup.h
View File

@ -345,27 +345,19 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk,
* calling bpf_setsockopt on listener-sk will not make sense anyway,
* so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here.
*/
#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \
({ \
int __ret = 0; \
if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) \
__ret = __cgroup_bpf_run_filter_sock_ops(sk, \
sock_ops, \
CGROUP_SOCK_OPS); \
__ret; \
})
#define __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \
__cgroup_bpf_run_filter_sock_ops(sk, sock_ops, \
CGROUP_SOCK_OPS)
#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \
({ \
int __ret = 0; \
if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && (sock_ops)->sk) { \
typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \
if (__sk && sk_fullsock(__sk)) \
__ret = __cgroup_bpf_run_filter_sock_ops(__sk, \
sock_ops, \
CGROUP_SOCK_OPS); \
} \
__ret; \
#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \
({ \
int __ret = 0; \
typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \
if (__sk && sk_fullsock(__sk)) \
__ret = __cgroup_bpf_run_filter_sock_ops(__sk, \
sock_ops, \
CGROUP_SOCK_OPS); \
__ret; \
})
#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) \
@ -529,7 +521,7 @@ static inline int cgroup_bpf_struct_ops_attach(struct bpf_map *map,
#define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; })
#define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; })
#define BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; })
#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
#define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) ({ 0; })
#define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; })
#define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \

56
include/net/tcp.h
View File

@ -2891,7 +2891,7 @@ static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
* program loaded).
*/
#ifdef CONFIG_BPF
static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
static inline int __tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
{
struct bpf_sock_ops_kern sock_ops;
int ret;
@ -2908,7 +2908,7 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
if (nargs > 0)
memcpy(sock_ops.args, args, nargs * sizeof(*args));
ret = BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
ret = __BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
if (ret == 0)
ret = sock_ops.reply;
else
@ -2916,20 +2916,23 @@ static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
return ret;
}
static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
{
u32 args[2] = {arg1, arg2};
#define tcp_call_bpf(sk, op) \
({ \
int __ret = 0; \
if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) { \
__ret = __tcp_call_bpf(sk, op, 0, NULL); \
} \
__ret; \
})
return tcp_call_bpf(sk, op, 2, args);
}
static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
u32 arg3)
{
u32 args[3] = {arg1, arg2, arg3};
return tcp_call_bpf(sk, op, 3, args);
}
#define tcp_call_bpf_flag(sk, op, ...) \
do { \
if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && \
BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), op ## _FLAG)) { \
u32 __args[] = { __VA_ARGS__ }; \
__tcp_call_bpf(sk, op, ARRAY_SIZE(__args), __args); \
} \
} while (0)
static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk)
{
@ -2938,21 +2941,12 @@ static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk)
}
#else
static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args)
static inline int tcp_call_bpf(struct sock *sk, int op)
{
return -EPERM;
}
static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2)
{
return -EPERM;
}
static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2,
u32 arg3)
{
return -EPERM;
}
#define tcp_call_bpf_flag(sk, op, ...) do { } while (0)
static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk)
{
@ -3136,7 +3130,7 @@ static inline u32 tcp_timeout_init(struct sock *sk)
{
int timeout;
timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL);
timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT);
timeout = bpf_tcp_ops_call_int(timeout_init, timeout, sk);
if (timeout <= 0)
timeout = TCP_TIMEOUT_INIT;
@ -3147,7 +3141,7 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
{
int rwnd;
rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL);
rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT);
rwnd = bpf_tcp_ops_call_int(rwnd_init, rwnd, sk);
if (rwnd < 0)
rwnd = 0;
@ -3156,14 +3150,12 @@ static inline u32 tcp_rwnd_init_bpf(struct sock *sk)
static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk)
{
return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1);
return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN) == 1);
}
static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt)
{
if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG))
tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt);
tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt);
bpf_tcp_ops_call_flag(rtt, RTT, sk, mrtt, srtt);
}

2
net/ipv4/af_inet.c
View File

@ -226,7 +226,7 @@ int __inet_listen_sk(struct sock *sk, int backlog)
if (err)
return err;
tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL);
tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB);
bpf_tcp_ops_call(listen, sk);
}
return 0;

3
net/ipv4/tcp.c
View File

@ -2994,8 +2994,7 @@ void tcp_set_state(struct sock *sk, int state)
*/
BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED);
if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
tcp_call_bpf_flag(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
bpf_tcp_ops_call(set_state, sk, state);
switch (state) {

21
net/ipv4/tcp_input.c
View File

@ -146,14 +146,16 @@ EXPORT_SYMBOL_GPL(clean_acked_data_flush);
#ifdef CONFIG_CGROUP_BPF
static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
{
bool unknown_opt = tcp_sk(sk)->rx_opt.saw_unknown &&
BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG);
bool parse_all_opt = BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG);
struct bpf_sock_ops_kern sock_ops;
const struct tcp_sock *tp;
if (likely(!unknown_opt && !parse_all_opt))
if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS))
return;
tp = tcp_sk(sk);
if (!(tp->rx_opt.saw_unknown &&
BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG)) &&
!BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG))
return;
/* The skb will be handled in the
@ -176,7 +178,7 @@ static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)
sock_ops.sk = sk;
bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb));
BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
__BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
}
static void bpf_skops_established(struct sock *sk, int bpf_op,
@ -184,6 +186,9 @@ static void bpf_skops_established(struct sock *sk, int bpf_op,
{
struct bpf_sock_ops_kern sock_ops;
if (!cgroup_bpf_enabled(CGROUP_SOCK_OPS))
return;
sock_owned_by_me(sk);
memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
@ -195,7 +200,7 @@ static void bpf_skops_established(struct sock *sk, int bpf_op,
if (skb)
bpf_skops_init_skb(&sock_ops, skb, tcp_hdrlen(skb));
BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
__BPF_CGROUP_RUN_PROG_SOCK_OPS(&sock_ops);
}
#else
static void bpf_skops_parse_hdr(struct sock *sk, struct sk_buff *skb)

2
net/ipv4/tcp_nv.c
View File

@ -146,7 +146,7 @@ static void tcpnv_init(struct sock *sk)
* within a datacenter, where we have reasonable estimates of
* RTTs
*/
base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT, 0, NULL);
base_rtt = tcp_call_bpf(sk, BPF_SOCK_OPS_BASE_RTT);
if (base_rtt > 0) {
ca->nv_base_rtt = base_rtt;
ca->nv_lower_bound_rtt = (base_rtt * 205) >> 8; /* 80% */

36
net/ipv4/tcp_output.c
View File

@ -476,8 +476,8 @@ static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
struct bpf_sock_ops_kern sock_ops;
int err;
if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) ||
if (!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG) ||
!remaining)
return remaining;
@ -518,7 +518,7 @@ static u32 bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
if (skb)
bpf_skops_init_skb(&sock_ops, skb, 0);
err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
err = __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
if (err || sock_ops.remaining_opt_len == remaining)
return remaining;
@ -543,7 +543,8 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
first_opt_off = tcp_hdrlen(skb) - max_opt_len;
if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) &&
BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) {
struct bpf_sock_ops_kern sock_ops;
int err;
@ -567,7 +568,7 @@ static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
sock_ops.remaining_opt_len = max_opt_len;
bpf_skops_init_skb(&sock_ops, skb, first_opt_off);
err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
err = __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
if (!err)
nr_written = max_opt_len - sock_ops.remaining_opt_len;
}
@ -638,6 +639,11 @@ static __always_inline u32 tcp_bpf_hdr_opt_len(struct sock *sk, struct sk_buff *
struct tcp_out_options *opts,
u32 remaining)
{
if (cgroup_bpf_enabled(CGROUP_SOCK_OPS))
remaining = bpf_skops_hdr_opt_len(sk, skb, req, syn_skb,
synack_type, opts,
remaining);
if (cgroup_bpf_enabled(CGROUP_TCP_SOCK_OPS))
remaining = bpf_tcp_ops_hdr_opt_len(sk, skb, req, syn_skb,
synack_type, opts,
@ -1103,8 +1109,6 @@ static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
remaining -= tcp_options_fit_accecn(opts, 0, remaining);
}
remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
remaining);
remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
remaining);
@ -1193,8 +1197,6 @@ static unsigned int tcp_synack_options(const struct sock *sk,
remaining -= tcp_options_fit_accecn(opts, 0, remaining);
}
remaining = bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
synack_type, opts, remaining);
remaining = tcp_bpf_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
synack_type, opts, remaining);
@ -1293,15 +1295,6 @@ static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb
}
}
if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp,
BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) {
remaining = MAX_TCP_OPTION_SPACE - size;
remaining = bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
remaining);
size = MAX_TCP_OPTION_SPACE - remaining;
}
remaining = tcp_bpf_hdr_opt_len(sk, skb, NULL, NULL, 0, opts,
MAX_TCP_OPTION_SPACE - size);
@ -3738,9 +3731,8 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
}
if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG))
tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB,
TCP_SKB_CB(skb)->seq, segs, err);
tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RETRANS_CB,
TCP_SKB_CB(skb)->seq, segs, err);
bpf_tcp_ops_call(retrans, sk, skb, err);
if (unlikely(err) && err != -EBUSY)
@ -4368,7 +4360,7 @@ int tcp_connect(struct sock *sk)
struct sk_buff *buff;
int err;
tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL);
tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB);
bpf_tcp_ops_call(connect, sk);
#if defined(CONFIG_TCP_MD5SIG) && defined(CONFIG_TCP_AO)

6
net/ipv4/tcp_timer.c
View File

@ -286,10 +286,8 @@ static int tcp_write_timeout(struct sock *sk)
tcp_fastopen_active_detect_blackhole(sk, expired);
mptcp_active_detect_blackhole(sk, expired);
if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RTO_CB_FLAG))
tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RTO_CB,
icsk->icsk_retransmits,
icsk->icsk_rto, (int)expired);
tcp_call_bpf_flag(sk, BPF_SOCK_OPS_RTO_CB,
icsk->icsk_retransmits, icsk->icsk_rto, (int)expired);
bpf_tcp_ops_call(rto, sk);
if (expired) {