github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'bpf-fixes-for-the-verifier-arena-skb-and-lwt-helpers'

Browse Source 
Alexei Starovoitov says:

====================
bpf: Fixes for the verifier, arena, skb and lwt helpers

From: Alexei Starovoitov <ast@kernel.org>

These patches fix bugs in BPF:

  patches  area
  -------  ----------------------------------------------------------------
  01-02    stack traces: the limit of the trace length
  03-07    arena: the range of mmap() and of bpf_arena_free_pages()
  08-12    arguments of kfuncs and of extension progs
  13       cgroup detach: the permission check
  14       cgroup skb progs: the network offset
  15       bpf_skb_adjust_room(): the limit of gso_size
  16-24    lwt and seg6local End.BPF progs: pushed headers, skb->cb, dst
  25-27    skb offsets when a helper adds or removes room

10 of the patches are from Chris Mason and Josef Bacik. Patches 07 and 12
are selftests.

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
====================

Link: https://patch.msgid.link/20261101190535.1454146-1-alexei.starovoitov@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov 2026-11-02 00:10:40 +00:00
commit 714b711201
No known key found for this signature in database
16 changed files with 564 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Documentation/bpf/kfuncs.rst
View File

@ -407,6 +407,13 @@ referenced kptr (by invoking bpf_kptr_xchg). If not, the verifier fails the
loading of the BPF program until no lingering references remain in all possible
explored states of the program.
A pointer to a struct in the arguments of a KF_ACQUIRE kfunc accepts only an
object of that type. The same applies to the argument that a KF_RELEASE kfunc
releases. For other arguments a kfunc also accepts memory of the program, if the
struct has only scalar members. A struct without members, struct __sk_buff and
struct xdp_md never accept memory. A KF_ACQUIRE kfunc which wants such memory
has to pair the pointer with a __sz argument.
2.5.2 KF_RET_NULL flag
----------------------

10
include/linux/bpf.h
View File

@ -1752,12 +1752,22 @@ enum bpf_func_context {
BPF_FUNC_CTX_CNT,
};
struct bpf_subprog_arg_info {
enum bpf_arg_type arg_type;
union {
u32 mem_size;
u32 btf_id;
};
};
struct bpf_func_info_aux {
u16 linkage;
bool unreliable;
bool called[BPF_FUNC_CTX_CNT];
bool verified[BPF_FUNC_CTX_CNT];
bool called_with_pkt_ptrs;
/* what the callers of a global function are checked against */
struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
};
enum bpf_jit_poke_reason {

8
include/linux/bpf_verifier.h
View File

@ -923,14 +923,6 @@ struct bpf_func_ptr {
/* the subprog that a bpf_func_ptr pointed to was removed as dead code */
#define BPF_FUNC_PTR_DELETED ((u32)-1)
struct bpf_subprog_arg_info {
enum bpf_arg_type arg_type;
union {
u32 mem_size;
u32 btf_id;
};
};
enum priv_stack_mode {
PRIV_STACK_UNKNOWN,
NO_PRIV_STACK,

1
include/linux/btf.h
View File

@ -631,6 +631,7 @@ s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id);
int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt,
struct module *owner);
struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id);
bool btf_is_projection_type(const char *pname);
bool btf_is_projection_of(const char *pname, const char *tname);
bool btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf,
const struct btf_type *t, enum bpf_prog_type prog_type,

16
kernel/bpf/arena.c
View File

@ -634,7 +634,7 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad
if (pgoff)
return -EINVAL;
if (len > SZ_4G)
if (len > (u64)map->max_entries * PAGE_SIZE)
return -E2BIG;
/* if user_vm_start was specified at arena creation time */
@ -650,11 +650,11 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad
ret = mm_get_unmapped_area(filp, addr, len * 2, 0, flags);
if (IS_ERR_VALUE(ret))
return ret;
if (!ret)
/* user_vm_start == 0 means that the user range is not set yet */
return -EINVAL;
if ((ret >> 32) == ((ret + len - 1) >> 32))
return ret;
if (WARN_ON_ONCE(arena->user_vm_start))
/* checks at map creation time should prevent this */
return -EFAULT;
return round_up(ret, SZ_4G);
}
@ -679,7 +679,7 @@ static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
return -EBUSY;
/* Earlier checks should prevent this */
if (WARN_ON_ONCE(vma->vm_end - vma->vm_start > SZ_4G || vma->vm_pgoff))
if (WARN_ON_ONCE(vma_pages(vma) > map->max_entries || vma->vm_pgoff))
return -EFAULT;
if (remember_vma(arena, vma))
@ -988,6 +988,8 @@ static int arena_free_range(struct bpf_arena *arena, long uaddr, long page_cnt)
static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable)
{
/* The first mmap() may be setting the user range right now */
u64 user_vm_start = READ_ONCE(arena->user_vm_start);
struct mem_cgroup *new_memcg, *old_memcg;
u64 full_uaddr, uaddr_end;
struct arena_free_span *s;
@ -995,8 +997,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
/* only aligned lower 32-bit are relevant */
uaddr = (u32)uaddr;
uaddr &= PAGE_MASK;
full_uaddr = clear_lo32(arena->user_vm_start) + uaddr;
if (full_uaddr < arena->user_vm_start)
full_uaddr = clear_lo32(user_vm_start) + uaddr;
if (!user_vm_start || full_uaddr < user_vm_start)
return;
uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT));
if (full_uaddr >= uaddr_end)

19
kernel/bpf/btf.c
View File

@ -6590,6 +6590,15 @@ static int find_kern_ctx_type_id(enum bpf_prog_type prog_type)
return ctx_type->type;
}
/*
* A projection type is the view that programs have of a kernel object with
* another layout: see btf_is_projection_of() for the two types behind them.
*/
bool btf_is_projection_type(const char *pname)
{
return strcmp(pname, "__sk_buff") == 0 || strcmp(pname, "xdp_md") == 0;
}
bool btf_is_projection_of(const char *pname, const char *tname)
{
if (strcmp(pname, "__sk_buff") == 0 && strcmp(tname, "sk_buff") == 0)
@ -8483,9 +8492,8 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log,
}
/* This is an optional check to make program writing easier.
* Compare names of structs and report an error to the user.
* btf_prepare_func_args() already checked that t2 struct
* is a context type. btf_prepare_func_args() will check
* later that t1 struct is a context type as well.
* check_ext_args() compares what btf_prepare_func_args()
* derives for the two arguments.
*/
s1 = btf_name_by_offset(btf1, t1->name_off);
s2 = btf_name_by_offset(btf2, t2->name_off);
@ -9019,6 +9027,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
return err;
sub->arg_slot_cnt = slots_used;
/* check_ext_args() needs them after env is freed */
if (is_global)
memcpy(subprog_aux(env, subprog)->args, sub->args,
sizeof(subprog_aux(env, subprog)->args));
sub->args_cached = true;
return 0;

26
kernel/bpf/cgroup.c
View File

@ -1279,9 +1279,14 @@ static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs,
return ERR_PTR(-ENOENT);
/* to maintain backward compatibility NONE and OVERRIDE cgroups
* allow detaching with invalid FD (prog==NULL) in legacy mode
* allow detaching with invalid FD (prog==NULL) in legacy mode.
* Without the FD of the attached program the caller needs
* CAP_NET_ADMIN or CAP_SYS_ADMIN in the initial user namespace.
*/
return hlist_entry(progs->first, typeof(*pl), node);
pl = hlist_entry(progs->first, typeof(*pl), node);
if (pl->prog != prog && !bpf_net_capable())
return ERR_PTR(-EPERM);
return pl;
}
if (!prog && !link)
@ -1785,7 +1790,7 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
struct sk_buff *skb,
enum cgroup_bpf_attach_type atype)
{
unsigned int offset = -skb_network_offset(skb);
int offset = skb_network_offset(skb);
struct sock *save_sk;
void *saved_data_end;
struct cgroup *cgrp;
@ -1797,7 +1802,15 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
save_sk = skb->sk;
skb->sk = sk;
__skb_push(skb, offset);
/*
* The program sees the packet from its network header on, on
* whichever side of skb->data that header lies.
*/
if (offset > 0)
__skb_pull(skb, offset);
else
__skb_push(skb, -offset);
/* compute pointers for the bpf prog */
bpf_compute_and_save_data_end(skb, &saved_data_end);
@ -1839,7 +1852,10 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk,
ret = -EFAULT;
}
bpf_restore_data_end(skb, saved_data_end);
__skb_pull(skb, offset);
if (offset > 0)
__skb_push(skb, offset);
else
__skb_pull(skb, -offset);
skb->sk = save_sk;
return ret;

33
kernel/bpf/stackmap.c
View File

@ -522,12 +522,14 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map,
u32 skip = flags & BPF_F_SKIP_FIELD_MASK;
u32 max_depth;
max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
/* trace_nr can be above max_depth and max_depth below skip: clamp first */
trace_nr = min_t(u32, trace_nr, max_depth);
if (trace_nr <= skip)
/* skipping more than usable stack trace */
return -EFAULT;
max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags);
stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip);
stackid->nr = trace_nr - skip;
stackid->len = stackid->nr * sizeof(u64);
stackid->ips = trace->ip + skip;
stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0);
@ -642,7 +644,8 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
/* couldn't fetch the stack trace */
return -EFAULT;
err = stackid_fastpath(&stackid, map, trace, trace->nr, flags);
/* the entry is released already, an NMI can change trace->nr */
err = stackid_fastpath(&stackid, map, trace, READ_ONCE(trace->nr), flags);
if (err != -ENOENT)
return err;
@ -822,14 +825,20 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
crosstask, false, 0);
}
if (unlikely(!trace) || trace->nr < skip) {
if (may_fault)
rcu_read_unlock();
preempt_enable();
goto err_fault;
}
if (unlikely(!trace))
goto err_unlock;
trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags);
/*
* trace->nr can be above max_depth. The per-CPU entry is released
* already: a tracing program or an NMI which takes a callchain in the
* same context before the copy below reuses it. And the unwinders of
* PowerPC store context markers on top of max_depth.
*/
trace_nr = min_t(u32, READ_ONCE(trace->nr), max_depth);
if (trace_nr < skip)
goto err_unlock;
trace_nr = callchain_store(trace, trace_nr, buf, elem_size, flags);
/* trace should not be dereferenced after this point */
if (may_fault)
@ -838,6 +847,10 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault);
err_unlock:
if (may_fault)
rcu_read_unlock();
preempt_enable();
err_fault:
err = -EFAULT;
clear:

66
kernel/bpf/verifier.c
View File

@ -13574,7 +13574,12 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env,
if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID)
return 0;
/*
* A kfunc that releases an object, or that takes a reference to one,
* needs the object itself: memory of the program cannot stand in for it.
*/
if (!is_kfunc(meta) || arg_type_is_release(*arg_type) ||
is_kfunc_acquire(meta) ||
base_type(reg->type) == PTR_TO_BTF_ID ||
reg2btf_ids[base_type(reg->type)])
return 0;
@ -13585,7 +13590,15 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env,
ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id);
ref_tname = btf_name_by_offset(btf, ref_t->name_off);
if (!btf_type_is_scalar_struct(env, btf, ref_t))
/*
* A kfunc that takes a projection type (ie. xdp_md) casts it to the
* underlying type. So memory of the program cannot stand in for it,
* even if the projection type holds only scalars. A struct without
* members (ie. iou_ctx) is a handle for an object of another type.
*/
if (btf_is_projection_type(ref_tname) ||
!btf_type_is_scalar_struct(env, btf, ref_t) ||
!btf_type_vlen(ref_t))
return 0;
resolve_ret = btf_resolve_size(btf, ref_t, arg_size);
@ -14340,9 +14353,13 @@ static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_re
/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
* actually use it -- it must cast to the underlying type. So we allow
* caller to pass in the underlying type.
* caller to pass in the underlying type. Only kernel BTF has that type:
* a type of the same name in the BTF of the program is program memory.
* And only a pointer to the start of the object points to that type.
*/
taking_projection = is_kfunc(meta) && btf_is_projection_of(arg_tname, reg_tname);
taking_projection = is_kfunc(meta) && btf_is_kernel(reg_btf) &&
!reg->var_off.value &&
btf_is_projection_of(arg_tname, reg_tname);
if (!taking_projection && !struct_same) {
verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n",
meta->func_name, reg_arg_name(env, argno),
@ -22510,6 +22527,37 @@ static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog)
!sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK];
}
/*
* The callers of a global function are checked against the arguments that its
* BTF describes, and an extension is verified with those of its own BTF. It
* must not expect more than the callers of the function provide.
*
* While an extension is loaded, bpf_check_attach_target() runs before its
* arguments are known. check_attach_btf_id() does the check when they are.
*/
static int check_ext_args(struct bpf_verifier_log *log, const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog, int subprog)
{
const struct bpf_subprog_arg_info *tgt = tgt_prog->aux->func_info_aux[subprog].args;
const struct bpf_subprog_arg_info *ext = prog->aux->func_info_aux[0].args;
int i;
for (i = 0; i < MAX_BPF_FUNC_REG_ARGS && ext[i].arg_type != ARG_UNUSED; i++) {
/* An extension may accept NULL where the callers never pass it. */
if ((ext[i].arg_type == tgt[i].arg_type ||
ext[i].arg_type == (tgt[i].arg_type | PTR_MAYBE_NULL)) &&
(base_type(ext[i].arg_type) == ARG_PTR_TO_MEM ?
ext[i].mem_size <= tgt[i].mem_size : ext[i].btf_id == tgt[i].btf_id))
continue;
bpf_log(log, "Extension program's argument in R%d is incompatible with original\n",
i + 1);
return -EINVAL;
}
return 0;
}
int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog,
@ -22797,7 +22845,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
return -EINVAL;
}
if (prog_extension &&
btf_check_type_match(log, prog, btf, t))
(btf_check_type_match(log, prog, btf, t) ||
check_ext_args(log, prog, tgt_prog, subprog)))
return -EINVAL;
t = btf_attach_func_proto(log, btf, btf_id);
if (!t || !btf_type_is_func_proto(t))
@ -23017,6 +23066,15 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
*/
env->ops = bpf_verifier_ops[tgt_prog->type];
prog->expected_attach_type = tgt_prog->expected_attach_type;
/* only now it is known what the arguments of prog stand for */
ret = btf_prepare_func_args(env, 0);
if (ret)
return ret;
ret = check_ext_args(&env->log, prog, tgt_prog,
find_tgt_subprog(tgt_prog, btf_id));
if (ret)
return ret;
}
/* store info about the attachment target that will be used later */

3
net/bpf/test_run.c
View File

@ -1210,7 +1210,8 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
skb->ip_summed = CHECKSUM_COMPLETE;
}
if (prog->type == BPF_PROG_TYPE_LWT_XMIT) {
if (prog->type == BPF_PROG_TYPE_LWT_IN ||
prog->type == BPF_PROG_TYPE_LWT_XMIT) {
if (!ipv6_mod_enabled()) {
pr_warn_once("Please test this program with IPv6 enabled kernel\n");
ret = -EOPNOTSUPP;

133
net/core/filter.c
View File

@ -3369,6 +3369,10 @@ static const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len)
{
/* The bytes in front of @off are moved with skb->data. */
if (unlikely(off > skb->len || !pskb_may_pull(skb, off)))
return -ENOMEM;
/* Caller already did skb_cow() with meta_len+len as headroom,
* so no need to do it here.
*/
@ -3389,9 +3393,10 @@ static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
void *old_data;
/* skb_ensure_writable() is not needed here, as we're
* already working on an uncloned skb.
* already working on an uncloned skb. The first test
* keeps off + len from wrapping.
*/
if (unlikely(!pskb_may_pull(skb, off + len)))
if (unlikely(off > skb->len || !pskb_may_pull(skb, off + len)))
return -ENOMEM;
old_data = skb->data;
@ -3402,9 +3407,25 @@ static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
return 0;
}
/*
* bpf_skb_net_hdr_push() inserts bytes at @off, bpf_skb_net_hdr_pop() removes
* bytes at @off. Both move the bytes in front of @off together with skb->data,
* and leave the rest of the packet where it is. Return true if the transport
* header is in the part that moves. A transport header that is equal to the
* network header stays equal to it.
*/
static bool bpf_skb_trans_hdr_moves(const struct sk_buff *skb, u32 off)
{
if (skb->transport_header == skb->network_header)
return true;
return skb_transport_header_was_set(skb) &&
(u32)skb_transport_offset(skb) < off;
}
static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len)
{
bool trans_same = skb->transport_header == skb->network_header;
bool trans_moves = bpf_skb_trans_hdr_moves(skb, off);
int ret;
/* There's no need for __skb_push()/__skb_pull() pair to
@ -3415,8 +3436,8 @@ static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len)
if (likely(!ret)) {
skb->mac_header -= len;
skb->network_header -= len;
if (trans_same)
skb->transport_header = skb->network_header;
if (trans_moves)
skb->transport_header -= len;
}
return ret;
@ -3424,7 +3445,7 @@ static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len)
static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len)
{
bool trans_same = skb->transport_header == skb->network_header;
bool trans_moves = bpf_skb_trans_hdr_moves(skb, off);
int ret;
/* Same here, __skb_push()/__skb_pull() pair not needed. */
@ -3432,8 +3453,17 @@ static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len)
if (likely(!ret)) {
skb->mac_header += len;
skb->network_header += len;
if (trans_same)
skb->transport_header = skb->network_header;
if (trans_moves) {
skb->transport_header += len;
} else if (skb_transport_header_was_set(skb) &&
skb_transport_offset(skb) < (int)off) {
/* It was in the removed bytes or before skb->data. */
skb_set_transport_header(skb, off);
}
/* Do not leave the inner transport header in front of @off. */
if (skb->encapsulation &&
skb_inner_transport_offset(skb) < (int)off)
skb_set_inner_transport_header(skb, off);
}
return ret;
@ -3744,6 +3774,25 @@ static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff,
return 0;
}
/*
* Code that went by the inner transport header while skb->encapsulation was set
* goes by the transport header from now on. So take the inner one over, unless
* it is the smaller one, as it is if nobody has set it. Do not leave the result
* in front of @off, where the bytes were removed.
*/
static void bpf_skb_clear_encap(struct sk_buff *skb, u32 off)
{
int thoff;
if (!skb->encapsulation)
return;
thoff = max(skb->inner_transport_header, skb->transport_header) -
skb_headroom(skb);
skb_set_transport_header(skb, max_t(int, thoff, off));
skb->encapsulation = 0;
}
static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff,
u64 flags)
{
@ -3762,6 +3811,14 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff,
return -ENOTSUPP;
}
/*
* The MSS is raised by len_diff. It has 16 bits, and GSO_BY_FRAGS is
* not a size.
*/
if (skb_is_gso(skb) && !(flags & BPF_F_ADJ_ROOM_FIXED_GSO) &&
skb_shinfo(skb)->gso_size + len_diff >= GSO_BY_FRAGS)
return -ENOTSUPP;
ret = skb_unclone(skb, GFP_ATOMIC);
if (unlikely(ret < 0))
return ret;
@ -3817,8 +3874,7 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff,
SKB_GSO_IPXIP4 |
SKB_GSO_IPXIP6 |
SKB_GSO_ESP)))
if (skb->encapsulation)
skb->encapsulation = 0;
bpf_skb_clear_encap(skb, off);
}
/* Header must be checked, and gso_segs recomputed. */
@ -3826,10 +3882,9 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff,
shinfo->gso_segs = 0;
} else {
/* For non-GSO packets, clear encapsulation if decap flags are set */
if ((flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK |
BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) &&
skb->encapsulation)
skb->encapsulation = 0;
if (flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK |
BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK))
bpf_skb_clear_encap(skb, off);
}
return 0;
@ -6916,14 +6971,22 @@ static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len
int err;
struct ipv6_sr_hdr *srh = (struct ipv6_sr_hdr *)hdr;
/* With ipv6.disable=1 the netns has no seg6 data and no IPv6 tables. */
if (!ipv6_mod_enabled())
return -EAFNOSUPPORT;
if (!seg6_validate_srh(srh, len, false))
return -EINVAL;
/*
* The encap mode names IPv6 as the inner protocol, and neither mode
* updates skb->protocol.
*/
if (skb->protocol != htons(ETH_P_IPV6))
return -EBADMSG;
switch (type) {
case BPF_LWT_ENCAP_SEG6_INLINE:
if (skb->protocol != htons(ETH_P_IPV6))
return -EBADMSG;
err = seg6_do_srh_inline(skb, srh);
break;
case BPF_LWT_ENCAP_SEG6:
@ -7012,12 +7075,13 @@ BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset,
this_cpu_ptr(&seg6_bpf_srh_states);
struct ipv6_sr_hdr *srh = srh_state->srh;
void *srh_tlvs, *srh_end, *ptr;
int srhoff = 0;
int srhoff;
lockdep_assert_held(&srh_state->bh_lock);
if (srh == NULL)
return -EINVAL;
srhoff = (unsigned char *)srh - skb->data;
srh_tlvs = (void *)((char *)srh + ((srh->first_segment + 1) << 4));
srh_end = (void *)((char *)srh + sizeof(*srh) + srh_state->hdrlen);
@ -7030,8 +7094,7 @@ BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset,
if (unlikely(bpf_try_make_writable(skb, offset + len)))
return -EFAULT;
if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0)
return -EINVAL;
/* The head may have moved. */
srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
memcpy(skb->data + offset, from, len);
@ -7147,16 +7210,18 @@ BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset,
{
struct seg6_bpf_srh_state *srh_state =
this_cpu_ptr(&seg6_bpf_srh_states);
bool mac_set = skb_mac_header_was_set(skb);
struct ipv6_sr_hdr *srh = srh_state->srh;
void *srh_end, *srh_tlvs, *ptr;
struct ipv6hdr *hdr;
int srhoff = 0;
int srhoff;
int ret;
lockdep_assert_held(&srh_state->bh_lock);
if (unlikely(srh == NULL))
return -EINVAL;
srhoff = (unsigned char *)srh - skb->data;
srh_tlvs = (void *)((unsigned char *)srh + sizeof(*srh) +
((srh->first_segment + 1) << 4));
srh_end = (void *)((unsigned char *)srh + sizeof(*srh) +
@ -7167,26 +7232,46 @@ BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset,
return -EFAULT;
if (unlikely(len < 0 && (void *)((char *)ptr - len) > srh_end))
return -EFAULT;
/* The length field of the SRH has 8 bits, in units of 8 bytes. */
if (unlikely(len > (U8_MAX << 3) - srh_state->hdrlen))
return -EFAULT;
/*
* skb->data is the network header here. The MAC header is in front of
* it, so bpf_skb_net_hdr_push() and bpf_skb_net_hdr_pop() do not move
* it with the other headers.
*/
if (len > 0) {
ret = skb_cow_head(skb, len);
ret = skb_cow_head(skb, len + skb->mac_len);
if (unlikely(ret < 0))
return ret;
/* The push can write over the MAC header, so move it first. */
if (mac_set)
memmove(skb->data - len - skb->mac_len,
skb_mac_header(skb), skb->mac_len);
ret = bpf_skb_net_hdr_push(skb, offset, len);
} else {
ret = bpf_skb_net_hdr_pop(skb, offset, -1 * len);
/* The pop has added -len to skb->mac_header. */
if (likely(!ret) && mac_set)
memmove(skb->data - skb->mac_len,
skb_mac_header(skb) + len, skb->mac_len);
}
bpf_compute_data_pointers(skb);
if (unlikely(ret < 0))
return ret;
if (mac_set)
skb_set_mac_header(skb, -skb->mac_len);
else
skb_unset_mac_header(skb);
hdr = (struct ipv6hdr *)skb->data;
hdr->payload_len = htons(skb->len - sizeof(struct ipv6hdr));
if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0)
return -EINVAL;
/* The bytes in front of offset have moved together with skb->data. */
srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
srh_state->hdrlen += len;
srh_state->valid = false;

53
net/core/lwt_bpf.c
View File

@ -85,6 +85,32 @@ static int run_lwt_bpf(struct sk_buff *skb, struct bpf_lwt_prog *lwt,
return ret;
}
/*
* skb->cb still describes the IP header that was the first one before the
* prog ran. The option offsets in IPCB(skb)->opt are wrong for a pushed
* header, and a header of the other family needs the other layout. iif
* stays: behind a VRF skb->skb_iif is the L3 master and not the port.
* bpf_lwt_push_encap() cannot do this, because bpf_prog_run_save_cb() can
* restore skb->cb after the prog.
*/
static void bpf_lwt_reset_cb(struct sk_buff *skb, __be16 proto)
{
int iif;
if (skb->protocol == proto) {
if (proto == htons(ETH_P_IP))
memset(&IPCB(skb)->opt, 0, sizeof(IPCB(skb)->opt));
} else if (skb->protocol == htons(ETH_P_IP)) {
iif = IP6CB(skb)->iif;
memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
IPCB(skb)->iif = iif;
} else {
iif = IPCB(skb)->iif;
memset(IP6CB(skb), 0, sizeof(*IP6CB(skb)));
IP6CB(skb)->iif = iif;
}
}
static int bpf_lwt_input_reroute(struct sk_buff *skb)
{
enum skb_drop_reason reason;
@ -102,7 +128,7 @@ static int bpf_lwt_input_reroute(struct sk_buff *skb)
dev_put(dev);
} else if (skb->protocol == htons(ETH_P_IPV6)) {
skb_dst_drop(skb);
if (IS_ENABLED(CONFIG_IPV6)) {
if (ipv6_mod_enabled()) {
ip6_route_input(skb);
err = skb_dst(skb)->error;
} else {
@ -124,6 +150,8 @@ static int bpf_lwt_input_reroute(struct sk_buff *skb)
static int bpf_input(struct sk_buff *skb)
{
struct dst_entry *dst = skb_dst(skb);
__be16 proto = skb->protocol;
unsigned int len = skb->len;
struct bpf_lwt *bpf;
int ret;
@ -132,8 +160,21 @@ static int bpf_input(struct sk_buff *skb)
ret = run_lwt_bpf(skb, &bpf->in, dst, NO_REDIRECT);
if (ret < 0)
return ret;
/* An lwt_in prog can change the length only with a push. */
if (skb->len != len)
bpf_lwt_reset_cb(skb, proto);
if (ret == BPF_LWT_REROUTE)
return bpf_lwt_input_reroute(skb);
/*
* orig_input() is for the old protocol only, see bpf_xmit().
* It also needs a dst of the old family. BPF_LWT_ENCAP_SEG6
* puts an IPv6 dst into the skb.
*/
if (skb->protocol != proto ||
skb_dst(skb)->ops->family != dst->ops->family) {
kfree_skb(skb);
return -EINVAL;
}
}
if (unlikely(!dst->lwtstate->orig_input)) {
@ -311,6 +352,8 @@ static int bpf_xmit(struct sk_buff *skb)
case BPF_REDIRECT:
return LWTUNNEL_XMIT_DONE;
case BPF_LWT_REROUTE:
/* An lwt_xmit prog can also write to the options. */
bpf_lwt_reset_cb(skb, proto);
return bpf_lwt_xmit_reroute(skb);
default:
return ret;
@ -649,6 +692,14 @@ int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress)
if (ingress)
skb_postpush_rcsum(skb, iph, len);
skb_reset_network_header(skb);
/*
* The new header goes over the MAC header. On ingress move it first.
* On egress the packet gets a new one, so forget the old one.
*/
if (ingress)
skb_mac_header_rebuild(skb);
else
skb_unset_mac_header(skb);
if (is_udp_tunnel) {
size_t iph_sz = ipv4 ? iph->ihl * 4 : sizeof(struct ipv6hdr);

37
tools/testing/selftests/bpf/prog_tests/arena_mmap.c Normal file
View File

@ -0,0 +1,37 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include <sys/mman.h>
#define ARENA_PAGES 8
static void test_refused(int pages, int flags, int err)
{
LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
void *area;
int fd;
fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, ARENA_PAGES, &opts);
if (fd == -EOPNOTSUPP) {
test__skip();
return;
}
if (!ASSERT_GE(fd, 0, "bpf_map_create"))
return;
area = mmap(NULL, pages * getpagesize(), PROT_READ | PROT_WRITE,
MAP_SHARED | flags, fd, 0);
if (ASSERT_EQ(area, MAP_FAILED, "mmap"))
ASSERT_EQ(errno, err, "errno");
else
munmap(area, pages * getpagesize());
close(fd);
}
void test_arena_mmap(void)
{
if (test__start_subtest("long"))
test_refused(ARENA_PAGES + 1, 0, E2BIG);
if (test__start_subtest("address_0"))
test_refused(ARENA_PAGES, MAP_FIXED, EINVAL);
}

92
tools/testing/selftests/bpf/prog_tests/freplace_args.c Normal file
View File

@ -0,0 +1,92 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include "freplace_args.skel.h"
#include "freplace_args_tgt.skel.h"
/*
* An extension must not expect more of its arguments than the callers of the
* function that it replaces provide. The progs are loaded and attached to each
* other. None of them runs.
*/
static const struct test {
const char *name;
const char *ext; /* prog of freplace_args.c */
const char *tgt; /* prog of freplace_args_tgt.c */
const char *func; /* function of tgt that ext is loaded for */
const char *func2; /* function of tgt that ext is attached to */
int bad_reg; /* argument that is refused, 0: none */
} tests[] = {
{ "same_nonnull", "ext_mem_nonnull", "tc_prog", "mem_nonnull" },
{ "same_trusted", "ext_task_trusted", "tc_prog", "task_trusted" },
{ "smaller_struct", "ext_mem_smaller", "tc_prog", "mem_smaller" },
{ "larger_struct", "ext_mem_larger", "tc_prog", "mem_larger", NULL, 1 },
{ "nullable_for_nonnull", "ext_mem", "tc_prog", "mem_nonnull" },
{ "nonnull_for_nullable", "ext_mem_nonnull", "tc_prog", "mem", NULL, 2 },
{ "trusted_for_mem", "ext_task_trusted", "tc_prog", "task_mem", NULL, 1 },
{ "arg_ctx_for_ctx_type", "ext_skb_ctx", "tc_prog", "skb_ctx" },
{ "mem_for_arg_ctx", "ext_tp_mem", "tp_prog", "tagged_ctx", NULL, 1 },
{ "attach_nullable_to_nonnull", "ext_mem", "tc_prog", "mem", "mem_nonnull" },
{ "attach_nonnull_to_nullable", "ext_mem_nonnull", "tc_prog", "mem_nonnull", "mem", 2 },
};
static void run_test(struct freplace_args_tgt *tgt_skel, const struct test *t)
{
struct bpf_program *ext, *tgt;
struct freplace_args *skel;
char log[4096], msg[128];
struct bpf_link *link;
int err;
skel = freplace_args__open();
if (!ASSERT_OK_PTR(skel, "open"))
return;
ext = bpf_object__find_program_by_name(skel->obj, t->ext);
tgt = bpf_object__find_program_by_name(tgt_skel->obj, t->tgt);
if (!ASSERT_OK_PTR(ext, "ext") || !ASSERT_OK_PTR(tgt, "tgt"))
goto out;
log[0] = '\0';
bpf_program__set_autoload(ext, true);
/* libbpf prints the log of a load that fails and must not */
if (t->bad_reg && !t->func2)
bpf_program__set_log_buf(ext, log, sizeof(log));
err = bpf_program__set_attach_target(ext, bpf_program__fd(tgt), t->func);
if (!ASSERT_OK(err, "set_attach_target"))
goto out;
err = freplace_args__load(skel);
if (t->bad_reg && !t->func2) {
snprintf(msg, sizeof(msg),
"Extension program's argument in R%d is incompatible with original",
t->bad_reg);
if (ASSERT_ERR(err, "load"))
ASSERT_HAS_SUBSTR(log, msg, "log");
goto out;
}
if (!ASSERT_OK(err, "load") || !t->func2)
goto out;
link = bpf_program__attach_freplace(ext, bpf_program__fd(tgt), t->func2);
ASSERT_EQ(libbpf_get_error(link), t->bad_reg ? -EINVAL : 0, "attach");
bpf_link__destroy(link);
out:
freplace_args__destroy(skel);
}
void test_freplace_args(void)
{
struct freplace_args_tgt *tgt_skel;
int i;
tgt_skel = freplace_args_tgt__open_and_load();
if (!ASSERT_OK_PTR(tgt_skel, "tgt_open_and_load"))
return;
for (i = 0; i < ARRAY_SIZE(tests); i++) {
if (test__start_subtest(tests[i].name))
run_test(tgt_skel, &tests[i]);
}
freplace_args_tgt__destroy(tgt_skel);
}

53
tools/testing/selftests/bpf/progs/freplace_args.c Normal file
View File

@ -0,0 +1,53 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
/* In freplace_args_tgt.c each of them has 16 bytes. */
struct buf { __u64 v[2]; };
struct larger { __u64 v[4]; };
struct smaller { __u64 v[1]; };
struct tp_ctx { __u64 v[2]; };
SEC("?freplace")
int ext_mem(int a, struct buf *p)
{
return 0;
}
SEC("?freplace")
int ext_mem_nonnull(int a, struct buf *p __arg_nonnull)
{
return 0;
}
SEC("?freplace")
int ext_mem_larger(struct larger *p)
{
return 0;
}
SEC("?freplace")
int ext_mem_smaller(struct smaller *p)
{
return 0;
}
SEC("?freplace")
int ext_task_trusted(struct task_struct *t __arg_trusted)
{
return 0;
}
SEC("?freplace")
int ext_skb_ctx(struct __sk_buff *skb __arg_ctx)
{
return 0;
}
SEC("?freplace")
int ext_tp_mem(struct tp_ctx *p)
{
return 0;
}
char _license[] SEC("license") = "GPL";

70
tools/testing/selftests/bpf/progs/freplace_args_tgt.c Normal file
View File

@ -0,0 +1,70 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
/* In freplace_args.c 'larger' has 32 bytes and 'smaller' has 8 bytes. */
struct buf { __u64 v[2]; };
struct larger { __u64 v[2]; };
struct smaller { __u64 v[2]; };
struct tp_ctx { __u64 v[2]; };
__weak int mem(int a, struct buf *p)
{
return 0;
}
__weak int mem_nonnull(int a, struct buf *p __arg_nonnull)
{
return 0;
}
__weak int mem_larger(struct larger *p)
{
return 0;
}
__weak int mem_smaller(struct smaller *p)
{
return 0;
}
/* No tag: memory of the size of the struct. */
__weak int task_mem(struct task_struct *t)
{
return 0;
}
__weak int task_trusted(struct task_struct *t __arg_trusted)
{
return 0;
}
__weak int skb_ctx(struct __sk_buff *skb)
{
return 0;
}
SEC("tc")
int tc_prog(struct __sk_buff *skb)
{
struct smaller smaller = {};
struct larger larger = {};
struct buf buf = {};
return mem(0, &buf) + mem_nonnull(0, &buf) + mem_larger(&larger) + mem_smaller(&smaller) +
task_mem(NULL) + task_trusted(bpf_get_current_task_btf()) + skb_ctx(skb);
}
/* In a tracepoint prog the tag makes a context of any struct. */
__weak int tagged_ctx(struct tp_ctx *ctx __arg_ctx)
{
return 0;
}
SEC("tracepoint")
int tp_prog(void *ctx)
{
return tagged_ctx(ctx);
}
char _license[] SEC("license") = "GPL";