diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 506c4201fec8..1d5f4cf88d33 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2284,8 +2284,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr if (ctx.offset == NULL) goto out_off; - ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); - ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + } out_cnt = bpf_out_stack_arg_cnt(env, prog); if (out_cnt) { diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index e78eb582c300..9393e4b0db8e 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -2245,8 +2245,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr memset(&ctx, 0, sizeof(ctx)); ctx.prog = prog; - ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); - ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + } ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); if (ctx.offset == NULL) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7a612688e7a2..0d6ea6df0106 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -241,8 +241,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr /* Make sure that the stack is quadword aligned. */ cgctx.stack_size = round_up(fp->aux->stack_depth, 16); - cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); - cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + if (bpf_prog_uses_arena_insns(fp)) { + cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); + cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + } cgctx.is_subprog = bpf_is_subprog(fp); cgctx.exception_boundary = fp->aux->exception_boundary; cgctx.exception_cb = fp->aux->exception_cb; diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 5265dd5bd39e..f2497ec7d378 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -73,8 +73,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr goto skip_init_ctx; } - ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); - ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + if (bpf_prog_uses_arena_insns(prog)) { + ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); + ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + } stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog); /* First 3 stack args in regs, rest on stack */ diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 14c905697352..8eaaac7028a1 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2304,10 +2304,11 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp, else jit->frame_off = 0; - kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); - if (kern_arena) + if (bpf_prog_uses_arena_insns(fp)) { + kern_arena = bpf_arena_get_kern_vm_start(fp->aux->arena); jit->kern_arena = _EMIT_CONST_U64(kern_arena); - jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); + jit->user_arena = bpf_arena_get_user_vm_start(fp->aux->arena); + } bpf_jit_prologue(jit, fp); if (bpf_set_addr(jit, 0) < 0) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d3843045dc71..72a2a0a89e03 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -888,7 +888,7 @@ static void emit_bpf_tail_call_indirect(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); - if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + if (bpf_prog_uses_arena_insns(bpf_prog)) pop_r12(&prog); } @@ -954,7 +954,7 @@ static void emit_bpf_tail_call_direct(struct bpf_prog *bpf_prog, pop_r12(&prog); } else { pop_callee_regs(&prog, callee_regs_used); - if (bpf_arena_get_kern_vm_start(bpf_prog->aux->arena)) + if (bpf_prog_uses_arena_insns(bpf_prog)) pop_r12(&prog); } @@ -2443,7 +2443,7 @@ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, if (!(flags & BTF_FMODEL_ARENA_ARG)) continue; - if (WARN_ON_ONCE(!bpf_prog->aux->arena)) + if (WARN_ON_ONCE(!bpf_prog_uses_arena_insns(bpf_prog))) return -EINVAL; /* mov eN, eN: truncate and clear the upper 32 bits */ @@ -2522,7 +2522,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY]; void __percpu *priv_frame_ptr = NULL; u16 out_stack_arg_cnt, outgoing_rsp; - u64 arena_vm_start, user_vm_start; + u64 arena_vm_start = 0, user_vm_start = 0; void __percpu *priv_stack_ptr; int i, excnt = 0; int ilen, proglen = 0; @@ -2562,8 +2562,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * [below] callee-saved regs * [below] outgoing args 7+ (= rsp) */ - arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); - user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); + if (bpf_prog_uses_arena_insns(bpf_prog)) { + arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena); + user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena); + } detect_reg_usage(insn, insn_cnt, callee_regs_used); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index dae33b1a7427..7750df973494 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1898,6 +1898,8 @@ struct bpf_prog_aux { bool has_prologue; bool might_sleep; bool kprobe_write_ctx; + /* has insns that work with the arena base, see bpf_prog_has_arena_insns() */ + bool uses_arena_insns; struct { s32 keyring_serial; u8 keyring_type; diff --git a/include/linux/bpf_compiler_types.h b/include/linux/bpf_compiler_types.h new file mode 100644 index 000000000000..5c9e71acfdac --- /dev/null +++ b/include/linux/bpf_compiler_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_BPF_COMPILER_TYPES_H +#define __LINUX_BPF_COMPILER_TYPES_H + +#include + +/* + * Qualifiers for the BPF verifier rather than address spaces: __arena marks a + * pointer into a BPF arena, __nullable a pointer that may be NULL. They reach + * BPF programs as BTF type tags on the members of kernel structs shared with + * them; to kernel code and to sparse the pointers are ordinary. + */ +#ifdef __CHECKER__ +# define __arena +# define __nullable +#else +# define __arena BTF_TYPE_TAG(arena) +# define __nullable BTF_TYPE_TAG(nullable) +#endif + +#endif /* __LINUX_BPF_COMPILER_TYPES_H */ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 951a90458d18..54481da4ed2d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -793,6 +793,14 @@ struct bpf_insn_aux_data { */ u64 fastcall_spills_num:3; u64 arg_prog:4; + /* + * What a load found, on any path: an __arena pointer member of a + * kernel struct, which arena_member_fixup() rewrites, one that is + * __nullable as well, or anything else through any pointer. + */ + u64 arena_member:1; + u64 arena_member_nullable:1; + u64 non_arena_load:1; /* below flags are initialized once */ u64 jmp_point:1; diff --git a/include/linux/filter.h b/include/linux/filter.h index 9339c6131f8f..6f4c9bd024a0 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1110,6 +1110,19 @@ static inline unsigned int bpf_prog_size(unsigned int proglen) offsetof(struct bpf_prog, insns[proglen])); } +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog); + +/* + * Whether a JIT has to set up the arena base for @prog. A program bound to + * an arena need not touch it, libbpf binds the arena of an object to every + * program in it. The flag caches bpf_prog_has_arena_insns() for the function + * about to be JITed, see bpf_prog_jit_compile() and jit_subprogs(). + */ +static inline bool bpf_prog_uses_arena_insns(const struct bpf_prog *prog) +{ + return prog->aux->uses_arena_insns; +} + static inline bool bpf_prog_was_classic(const struct bpf_prog *prog) { /* When classic BPF programs have been loaded and the arch diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a9bf4c3c0a27..538129ed2418 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3569,7 +3569,8 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } -bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +/* Whether the pointee of pointer type @t carries the type tag @name */ +static bool btf_ptr_has_type_tag(const struct btf *btf, const struct btf_type *t, const char *name) { if (!btf_type_is_ptr(t)) return false; @@ -3578,13 +3579,18 @@ bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) t = btf_type_by_id(btf, t->type)) { if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) continue; - if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena")) + if (!strcmp(__btf_name_by_offset(btf, t->name_off), name)) return true; } return false; } +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + return btf_ptr_has_type_tag(btf, t, "arena"); +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -7701,6 +7707,7 @@ enum bpf_struct_walk_result { WALK_SCALAR = 0, WALK_PTR, WALK_PTR_UNTRUSTED, + WALK_PTR_ARENA, WALK_STRUCT, }; @@ -7973,6 +7980,38 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, id = ctx.id; stype = ctx.t; + /* + * __nullable combines with any of the above: it only says + * that the pointer may be NULL. The caller decides whether + * that matters for the register type it derives. + */ + if (btf_ptr_has_type_tag(btf, mtype, "nullable")) + tmp_flag |= PTR_MAYBE_NULL; + + /* + * An __arena member of a kernel struct holds the kernel + * address of arena memory, which the verifier turns into + * an arena pointer on load. Program BTF is not affected: + * there the tag is what the compiler emits for the arena + * address space. An offset into a union resolves to its + * first member, so a program reading another member there + * would get an arena pointer it did not ask for; the walk + * marked such a union untrusted. + */ + if (btf_is_kernel(btf) && btf_type_is_arena_ptr(btf, mtype)) { + if (*flag & PTR_UNTRUSTED) { + bpf_log(log, + "arena pointer member %s of %s is inside a union\n", + mname, tname); + return -EACCES; + } + *next_btf_id = id; + *flag |= tmp_flag & PTR_MAYBE_NULL; + if (field_name) + *field_name = mname; + return WALK_PTR_ARENA; + } + if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; @@ -8075,6 +8114,11 @@ int btf_struct_access(struct bpf_verifier_log *log, *flag = MEM_RDONLY | PTR_UNTRUSTED; ret = PTR_TO_MEM; goto check_variable_offset; + case WALK_PTR_ARENA: + *next_btf_id = id; + *flag = tmp_flag; + ret = PTR_TO_ARENA; + goto check_variable_offset; case WALK_SCALAR: ret = SCALAR_VALUE; goto check_variable_offset; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6e32db9777c2..fc22d66d194a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2637,11 +2637,68 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) return select_interpreter; } +/* + * Whether a function has an insn that works with the arena base, which a JIT + * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on + * the final insns of the function: an access through an arena pointer, or + * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been + * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address + * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, + * which the JIT rebases, has a function model that says so. A load of an + * __arena struct member is rewritten with an immediate and needs none; + * dereferencing the result does. + */ +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog) +{ + const struct bpf_insn *insn = prog->insnsi; + const struct btf_func_model *m; + int i, j; + + if (!prog->aux->arena) + return false; + + for (i = 0; i < prog->len; i++, insn++) { + switch (BPF_CLASS(insn->code)) { + case BPF_LDX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) + return true; + break; + case BPF_ST: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) + return true; + break; + case BPF_STX: + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) + return true; + break; + case BPF_ALU64: + if (BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X && + insn->off == BPF_ADDR_SPACE_CAST) + return true; + break; + case BPF_JMP: + if (!bpf_pseudo_kfunc_call(insn)) + break; + m = bpf_jit_find_kfunc_model(prog, insn); + if (!m) + break; + for (j = 0; j < m->nr_args; j++) + if (m->arg_flags[j] & BTF_FMODEL_ARENA_ARG) + return true; + break; + } + } + return false; +} + static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; + prog->aux->uses_arena_insns = bpf_prog_has_arena_insns(prog); if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index bfbb0cf5c0a8..409b9104c723 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -882,6 +882,37 @@ static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) * struct __sk_buff -> struct sk_buff * struct bpf_sock_ops -> struct sock */ +/* + * A load through an __arena pointer member of a kernel struct reads the kernel + * address of arena memory. Append the conversion to the arena pointer form, + * (u32)(kaddr - kern_vm_start), that the struct_ops trampoline applies to + * __arena arguments. A __nullable member keeps NULL as NULL, tested on the + * full kernel address, and so does a probed load, which reads as 0 when it + * faults. Returns the number of insns in @buf, 0 when the insn loads no such + * member, or an error when it does so on some paths only. + */ +static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn, + struct bpf_insn *buf) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; + int cnt = 0; + u32 base; + + if (!aux->arena_member) + return 0; + if (aux->non_arena_load) { + verbose(env, "an insn loads an arena pointer member on one path and something else on another\n"); + return -EINVAL; + } + + base = bpf_arena_get_kern_vm_start(env->prog->aux->arena); + buf[cnt++] = *insn; + if (aux->arena_member_nullable || BPF_MODE(insn->code) == BPF_PROBE_MEM) + buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, insn->dst_reg, 0, 1); + buf[cnt++] = BPF_ALU32_IMM(BPF_SUB, insn->dst_reg, (s32)base); + return cnt; +} + int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprogs = env->subprog_info; @@ -1026,6 +1057,21 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; continue; + } else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && + insn->imm == BPF_LOAD_ACQ && + env->insn_aux_data[i + delta].arena_member) { + /* an acquire load of an __arena member is rewritten like a plain load */ + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; } else if (insn->code == (BPF_JMP | BPF_EXIT) && epilogue_cnt && i + delta < subprogs[1].start) { @@ -1117,11 +1163,23 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_LDX | BPF_PROBE_MEMSX | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; - continue; - } - if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, - "access to a fault prone pointer is not rewritten as a probed one")) + } else if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) { return -EFAULT; + } + + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + if (!cnt) + continue; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; continue; } @@ -1390,6 +1448,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->jited_linfo = prog->aux->jited_linfo; func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; func[i]->aux->arena = prog->aux->arena; + func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]); func[i]->aux->used_maps = env->used_maps; func[i]->aux->used_map_cnt = env->used_map_cnt; num_exentries = 0; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d13868846823..015dc1bccebe 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6585,6 +6585,45 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, "__safe_trusted_or_null"); } +/* + * The member holds the kernel address of arena memory, which the rewritten + * load turns into an arena pointer, see arena_member_fixup(). NULL needs no + * tracking, PTR_TO_ARENA covers it. + */ +static int check_arena_member_load(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int value_regno, enum bpf_type_flag flag, const char *tname, + const char *field_name) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx]; + struct bpf_insn *insn; + + /* a helper reading the member as bytes needs no rewrite */ + if (value_regno < 0) + return 0; + if (!env->prog->aux->arena) { + verbose(env, + "arena pointer member %s of struct %s requires a program with an associated arena\n", + field_name, tname); + return -EINVAL; + } + /* only a plain load or an acquire load is rewritten */ + insn = &env->prog->insnsi[env->insn_idx]; + if (BPF_CLASS(insn->code) != BPF_LDX && + !(insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && insn->imm == BPF_LOAD_ACQ)) { + verbose(env, + "arena pointer member %s of struct %s can only be read with a plain or an acquire load\n", + field_name, tname); + return -EACCES; + } + + aux->arena_member = true; + if (flag & PTR_MAYBE_NULL) + aux->arena_member_nullable = true; + mark_reg_unknown(env, regs, value_regno); + regs[value_regno].type = PTR_TO_ARENA; + return 0; +} + static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, argno_t argno, int off, int size, @@ -6694,6 +6733,9 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, if (ret < 0) return ret; + if (ret == PTR_TO_ARENA) + return check_arena_member_load(env, regs, value_regno, flag, tname, field_name); + if (ret != PTR_TO_BTF_ID) { /* just mark; */ @@ -6754,6 +6796,16 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, clear_trusted_flags(&flag); } + /* + * A __nullable member only matters where NULL would otherwise be + * assumed away, which is anywhere but on a plain or an untrusted + * pointer: those are probed on a dereference, which tolerates NULL, and + * the BTF_TYPE_SAFE_*_OR_NULL lists never applied to them either. A + * per-CPU pointer counts, per_cpu_ptr() does not check for NULL. + */ + if (bpf_may_fault_on_deref(PTR_TO_BTF_ID | (flag & ~PTR_MAYBE_NULL))) + flag &= ~PTR_MAYBE_NULL; + if (atype == BPF_READ && value_regno >= 0) { ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) @@ -7465,6 +7517,17 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); + /* + * Paths through a kernel struct pointer and through other pointers + * may share the insn, see save_aux_ptr_type(). Record a load that did + * not produce an arena pointer from an __arena member, the only load + * through a kernel struct pointer that does, next to the one + * check_arena_member_load() records, so that arena_member_fixup() can + * tell when the paths disagree. + */ + if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID && + regs[insn->dst_reg].type == PTR_TO_ARENA)) + env->insn_aux_data[env->insn_idx].non_arena_load = true; err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); @@ -20796,12 +20859,6 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, { enum bpf_prog_type prog_type = resolve_prog_type(prog); - if (map->excl_prog_sha && - memcmp(map->excl_prog_sha, prog->digest, SHA256_DIGEST_SIZE)) { - verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); - return -EACCES; - } - if (btf_record_has_field(map->record, BPF_LIST_HEAD) || btf_record_has_field(map->record, BPF_RB_ROOT)) { if (is_tracing_prog_type(prog_type)) { @@ -20899,6 +20956,61 @@ static int bpf_map_claim(struct bpf_verifier_env *env, struct bpf_map *map) } } +/* + * An exclusive map is bound to the digest of its program. The digest covers + * the insns as they are verified, after the CO-RE relocations the kernel + * applies, so it exists only once check_and_resolve_insns() has computed it, + * and the maps are checked against it there, whether an insn referred to + * them or a continuous fd_array bound them before. + */ +static int check_excl_maps(struct bpf_verifier_env *env) +{ + int i; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + if (map->excl_prog_sha && + memcmp(map->excl_prog_sha, env->prog->digest, SHA256_DIGEST_SIZE)) { + verbose(env, "program's hash doesn't match map's excl_prog_hash\n"); + return -EACCES; + } + } + return 0; +} + +/* + * The program may use every map it referred to, see check_excl_maps(). Only now + * does it leave a trace on them: it claims each as its user, see + * bpf_map_claim(), and takes over its insn arrays. A program rejected before + * this point, one that is not the owner of an exclusive map in particular, + * leaves the maps as it found them. Nothing consults a claim before + * resolve_func_ptrs() and the fixup passes, which run later. + */ +static int claim_used_maps(struct bpf_verifier_env *env) +{ + int i, err; + + for (i = 0; i < env->used_map_cnt; i++) { + struct bpf_map *map = env->used_maps[i]; + + err = bpf_map_claim(env, map); + if (err) + return err; + + if (map->map_type != BPF_MAP_TYPE_INSN_ARRAY) + continue; + err = bpf_insn_array_init(map, env->prog); + if (err) { + verbose(env, "Failed to properly initialize insn array\n"); + return err; + } + env->insn_array_maps[env->insn_array_map_cnt++] = map; + env->prog->jit_required = true; + } + return 0; +} + static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) { int i, err; @@ -20936,20 +21048,6 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map) env->used_maps[env->used_map_cnt++] = map; - err = bpf_map_claim(env, map); - if (err) - return err; - - if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) { - err = bpf_insn_array_init(map, env->prog); - if (err) { - verbose(env, "Failed to properly initialize insn array\n"); - return err; - } - env->insn_array_maps[env->insn_array_map_cnt++] = map; - env->prog->jit_required = true; - } - return env->used_map_cnt - 1; } @@ -21353,7 +21451,10 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) * 'struct bpf_map *' into a register instead of user map_fd. * These pointers will be used later by verifier to validate map access. */ - return 0; + err = check_excl_maps(env); + if (err) + return err; + return claim_used_maps(env); } /* that many pointers to functions are recognized in the maps of a program at most */ @@ -23619,11 +23720,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = -EINVAL; goto err_prep; } - if (env->signature) { - ret = bpf_prog_calc_tag(env->prog); - if (ret < 0) - goto err_prep; - } ret = process_fd_array(env, attr, uattr); if (ret) diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 996e61b5f11f..7dedd3975c39 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -1595,6 +1595,9 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, name = btf_name_of(d, t->name_off); if (btf_kflag(t)) btf_dump_printf(d, " __attribute__((%s))", name); + else if (strcmp(name, "arena") == 0) + /* the tag denotes a pointer into the BPF arena address space */ + btf_dump_printf(d, " __attribute__((address_space(1))) __attribute__((btf_type_tag(\"arena\")))"); else btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name); break; diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index 9f589980beef..adb5d2e7fdfa 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -77,6 +77,82 @@ static int probe_kern_global_data(int token_fd) return probe_fd(ret); } +/* + * A continuous fd_array binds every map in it to the program, whether an + * insn refers to the map or not. Kernels without the attribute reject a + * non-zero fd_array_cnt. + */ +static int probe_kern_fd_array_cnt(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .fd_array_cnt = 1, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), sizeof(int), 1, + &map_opts); + if (map < 0) { + ret = -errno; + pr_warn("Error in %s(): %s. Couldn't create simple array map.\n", + __func__, errstr(ret)); + return ret; + } + + prog_opts.fd_array = ↦ + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + +/* + * The kernel checks an exclusive map bound through fd_array_cnt against the + * digest of the final program. Earlier kernels compared it as the map was + * added, against a digest they had not computed yet, and refused the load for + * the very program the map is exclusive to; kernels without exclusive maps + * refuse the map. + */ +static int probe_kern_excl_map_fd_array_cnt(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + __u8 hash[SHA256_DIGEST_LENGTH]; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .excl_prog_hash = hash, + .excl_prog_hash_size = sizeof(hash), + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + .fd_array_cnt = 1, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + libbpf_sha256(insns, sizeof(insns), hash); + map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_excl", sizeof(int), sizeof(int), 1, + &map_opts); + if (map < 0) + return 0; + + prog_opts.fd_array = ↦ + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + static int probe_kern_btf(int token_fd) { static const char strs[] = "\0int"; @@ -742,6 +818,12 @@ static struct kern_feature_desc { [FEAT_PERCPU_DATA] = { "kernel supports percpu data", probe_kern_percpu_data, }, + [FEAT_FD_ARRAY_CNT] = { + "continuous fd_array (fd_array_cnt)", probe_kern_fd_array_cnt, + }, + [FEAT_EXCL_MAP_FD_ARRAY_CNT] = { + "exclusive map bound through fd_array_cnt", probe_kern_excl_map_fd_array_cnt, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index cb09ded90773..8fcb8cf39cc0 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -794,7 +794,13 @@ struct bpf_object { int *fd_array; size_t fd_array_cap; - size_t fd_array_cnt; + /* + * Slots handed out so far, slot 0 included. A kfunc insn names a module + * BTF slot through its offset, slot 0 stands for vmlinux BTF and stays + * empty, so the array goes to the kernel without a count, unless + * bpf_object__bind_arena() puts the arena of the object there. + */ + size_t fd_array_used; struct usdt_manager *usdt_man; @@ -8799,6 +8805,56 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); +/* + * Bind the arena of the object to the program about to be loaded. A program + * has an arena only when an insn refers to the map, but the kernel can hand + * it arena pointers all the same, as struct_ops arguments, through tagged + * struct members or as kfunc return values, and the verifier rejects those + * when the program has no arena to place them in. A continuous fd_array + * binds every map in it, the arena takes slot 0, which is free: a kfunc insn + * with offset 0 names vmlinux BTF rather than a slot. + */ +static int bpf_object__bind_arena(struct bpf_object *obj, struct bpf_program *prog, + struct bpf_prog_load_opts *load_attr) +{ + struct bpf_map *map; + int err; + + if (obj->gen_loader || obj->arena_map_idx < 0 || !kernel_supports(obj, FEAT_FD_ARRAY_CNT)) + return 0; + map = &obj->maps[obj->arena_map_idx]; + /* + * Nothing to bind when the arena is not created, or no longer is after + * a failed load, when it is exclusive to another program, and nothing + * for an offloaded program, which the kernel does not let near an + * arena. + */ + if (!map->autocreate || map->fd < 0 || (map->excl_prog && map->excl_prog != prog) || + load_attr->prog_ifindex) + return 0; + /* + * An arena exclusive to the program itself needs a kernel that checks + * the map against the digest of the final program. Earlier kernels + * compared a map bound through the fd_array against a digest they had + * not computed yet and refused the load; there the program binds its + * arena through its insns, as before. + */ + if (map->excl_prog && !kernel_supports(obj, FEAT_EXCL_MAP_FD_ARRAY_CNT)) + return 0; + + if (!obj->fd_array_used) { + err = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), 1); + if (err) + return err; + obj->fd_array_used = 1; + } + obj->fd_array[0] = map->fd; + load_attr->fd_array = obj->fd_array; + /* with slot 0 filled every slot handed out is valid, so the array can go with a count */ + load_attr->fd_array_cnt = obj->fd_array_used; + return 0; +} + static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, const char *license, __u32 kern_version, int *prog_fd) @@ -8862,6 +8918,9 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog if (obj->data_in_arena) load_attr.prog_flags |= BPF_F_ARENA_SCALAR; load_attr.fd_array = obj->fd_array; + err = bpf_object__bind_arena(obj, prog, &load_attr); + if (err) + return err; load_attr.token_fd = obj->token_fd; if (obj->token_fd) @@ -9716,22 +9775,22 @@ static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, /* set index for module BTF fd in fd_array, if unset */ if (mod_btf && !mod_btf->fd_array_idx) { /* insn->off is s16 */ - if (obj->fd_array_cnt == INT16_MAX) { + if (obj->fd_array_used == INT16_MAX) { pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n", ext->name, mod_btf->fd_array_idx); return -E2BIG; } /* Cannot use index 0 for module BTF fd */ - if (!obj->fd_array_cnt) - obj->fd_array_cnt = 1; + if (!obj->fd_array_used) + obj->fd_array_used = 1; ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), - obj->fd_array_cnt + 1); + obj->fd_array_used + 1); if (ret) return ret; - mod_btf->fd_array_idx = obj->fd_array_cnt; + mod_btf->fd_array_idx = obj->fd_array_used; /* we assume module BTF FD is always >0 */ - obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; + obj->fd_array[obj->fd_array_used++] = mod_btf->fd; } ext->is_set = true; @@ -9961,8 +10020,10 @@ static void bpf_object_cleanup_btf(struct bpf_object *obj) static void bpf_object_post_load_cleanup(struct bpf_object *obj) { - /* clean up fd_array */ + /* clean up fd_array, and the slots handed out with it */ zfree(&obj->fd_array); + obj->fd_array_cap = 0; + obj->fd_array_used = 0; /* clean up BTF */ bpf_object_cleanup_btf(obj); @@ -10940,7 +11001,13 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0) ?: prog->prog_ifindex; attr.kern_version = OPTS_GET(opts, kern_version, 0) ?: obj->kern_version; attr.fd_array = OPTS_GET(opts, fd_array, NULL) ?: obj->fd_array; - attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0) ?: obj->fd_array_cnt; + attr.fd_array_cnt = OPTS_GET(opts, fd_array_cnt, 0); + /* the arena of the object goes with the program, as in bpf_object__load() */ + if (!OPTS_GET(opts, fd_array, NULL) && !OPTS_GET(opts, fd_array_cnt, 0)) { + err = bpf_object__bind_arena(obj, prog, &attr); + if (err) + return libbpf_err(err); + } attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4e45dfacb548..e90b5336bac0 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -404,6 +404,10 @@ enum kern_feature_id { FEAT_BPF_SYSCALL_COMMON_ATTRS, /* Kernel supports percpu data */ FEAT_PERCPU_DATA, + /* Kernel binds every map in a continuous fd_array (fd_array_cnt) to the program */ + FEAT_FD_ARRAY_CNT, + /* Kernel checks an exclusive map bound through fd_array_cnt against the final digest */ + FEAT_EXCL_MAP_FD_ARRAY_CNT, __FEAT_CNT, }; diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 17fa44cb9c91..35f214e254da 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -604,8 +604,10 @@ static void test_btf_dump_type_tags(void) * struct s { * void __attribute__((btf_type_tag(\"void_tag\"))) *p1; * void __attribute__((void_attr)) *p2; + * void __arena *p3; * }; * + * The arena tag is rendered with the address space it denotes. */ id = btf__add_type_tag(btf, "void_tag", 0); @@ -618,17 +620,25 @@ static void test_btf_dump_type_tags(void) id = btf__add_ptr(btf, id); ASSERT_EQ(id, 4, "void_ptr_id2"); - id = btf__add_struct(btf, "s", 8); - ASSERT_EQ(id, 5, "struct_id"); + id = btf__add_type_tag(btf, "arena", 0); + ASSERT_EQ(id, 5, "arena_tag_id"); + id = btf__add_ptr(btf, id); + ASSERT_EQ(id, 6, "void_ptr_id3"); + + id = btf__add_struct(btf, "s", 24); + ASSERT_EQ(id, 7, "struct_id"); err = btf__add_field(btf, "p1", 2, 0, 0); ASSERT_OK(err, "field_ok1"); - err = btf__add_field(btf, "p2", 4, 0, 0); + err = btf__add_field(btf, "p2", 4, 64, 0); ASSERT_OK(err, "field_ok2"); + err = btf__add_field(btf, "p3", 6, 128, 0); + ASSERT_OK(err, "field_ok3"); test_ctx__dump_and_compare(&t, "struct s {\n" " void __attribute__((btf_type_tag(\"void_tag\"))) *p1;\n" " void __attribute__((void_attr)) *p2;\n" +" void __attribute__((address_space(1))) __attribute__((btf_type_tag(\"arena\"))) *p3;\n" "};\n\n", "dump_and_compare"); test_ctx__free(&t); diff --git a/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c b/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c new file mode 100644 index 000000000000..fa6ccbaa7fd9 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/clone_module_kfunc.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include "kfunc_call_test.skel.h" + +/* + * bpf_program__clone() loads a program from a prepared object with the + * fd_array of the object, where module BTF fds start at slot 1 and slot 0 is + * empty. It used to pass the fill index of that array as fd_array_cnt, which + * made the kernel resolve slot 0 and fail the load. + */ +void test_clone_module_kfunc(void) +{ + struct kfunc_call_test *skel; + int err, fd; + + skel = kfunc_call_test__open(); + if (!ASSERT_OK_PTR(skel, "skel_open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "prepare")) + goto out; + + fd = bpf_program__clone(skel->progs.kfunc_call_test1, NULL); + if (ASSERT_GE(fd, 0, "clone")) + close(fd); +out: + kfunc_call_test__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/map_excl.c b/tools/testing/selftests/bpf/prog_tests/map_excl.c index 3f4422b9ffa6..9ff4cd3de2ec 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_excl.c +++ b/tools/testing/selftests/bpf/prog_tests/map_excl.c @@ -161,12 +161,49 @@ static void test_map_excl_create_validation(void) ASSERT_EQ(fd, -EINVAL, "reject size with NULL hash pointer"); } +/* + * The exclusive map bound through a continuous fd_array rather than by an + * insn. The kernel resolves the array before it verifies the program, so it + * has to have the digest of the program at hand by then. + */ +static void test_map_excl_fd_array(void) +{ + struct map_excl *skel = map_excl__open(); + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int err, fd, map_fd; + + if (!ASSERT_OK_PTR(skel, "map_excl__open")) + return; + err = bpf_map__set_exclusive_program(skel->maps.excl_map, skel->progs.should_have_access); + if (!ASSERT_OK(err, "bpf_map__set_exclusive_program")) + goto out; + + bpf_program__set_autoload(skel->progs.should_have_access, true); + bpf_program__set_autoload(skel->progs.should_not_have_access, true); + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "map_excl prepare")) + goto out; + + map_fd = bpf_map__fd(skel->maps.excl_map); + opts.fd_array = &map_fd; + opts.fd_array_cnt = 1; + fd = bpf_program__clone(skel->progs.should_have_access, &opts); + if (ASSERT_GE(fd, 0, "exclusive program with the map in fd_array")) + close(fd); + fd = bpf_program__clone(skel->progs.should_not_have_access, &opts); + ASSERT_EQ(fd, -EACCES, "other program with the map in fd_array"); +out: + map_excl__destroy(skel); +} + void test_map_excl(void) { if (test__start_subtest("map_excl_allowed")) test_map_excl_allowed(); if (test__start_subtest("map_excl_denied")) test_map_excl_denied(); + if (test__start_subtest("map_excl_fd_array")) + test_map_excl_fd_array(); if (test__start_subtest("map_excl_no_map_in_map")) test_map_excl_no_map_in_map(); if (test__start_subtest("map_excl_no_map_iter")) diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c index 7f9f54ba3fbe..e7580b9f0fc2 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c @@ -5,6 +5,15 @@ #include "struct_ops_arena.skel.h" #include "struct_ops_arena_attach.skel.h" #include "struct_ops_arena_fail.skel.h" +#include "struct_ops_arena_member.skel.h" +#include "struct_ops_arena_member_render.skel.h" +#include "struct_ops_arena_member_fail__union.skel.h" +#include "struct_ops_arena_member_fail__xchg.skel.h" +#include "struct_ops_arena_member_fail__mismatch.skel.h" +#include "struct_ops_arena_member_fail__mismatch_mem.skel.h" +#include "struct_ops_arena_member_fail__write.skel.h" +#include "struct_ops_arena_member_fail__no_arena.skel.h" +#include "struct_ops_arena_autobind.skel.h" #if defined(__x86_64__) || defined(__aarch64__) /* @@ -101,6 +110,167 @@ static void arena_arg_attach(void) struct_ops_arena__destroy(skel); } + +/* Attach @st_ops and run @trigger once, it returns 0 when everything checked out. */ +static void attach_and_trigger(struct bpf_map *st_ops, struct bpf_program *trigger) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_link *link; + int err; + + link = bpf_map__attach_struct_ops(st_ops); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + return; + + err = bpf_prog_test_run_opts(bpf_program__fd(trigger), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + + bpf_link__destroy(link); +} + +/* + * Attach a callback that reads arena pointers out of its argument struct. + * The header is shared with the module, so the members are arena pointers + * on the program side as well. + */ +static void arena_member(void) +{ + struct struct_ops_arena_member *skel; + + skel = struct_ops_arena_member__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_member__open_and_load")) + return; + + attach_and_trigger(skel->maps.testmod_arena_member, skel->progs.trigger); + struct_ops_arena_member__destroy(skel); +} + +/* @prog_fd lists exactly one map, the one with @map_id. */ +static void assert_prog_map(int prog_fd, __u32 map_id, const char *name) +{ + __u32 map_ids[2] = {}, len = sizeof(struct bpf_prog_info); + struct bpf_prog_info info = { + .nr_map_ids = ARRAY_SIZE(map_ids), + .map_ids = ptr_to_u64(map_ids), + }; + int err; + + err = bpf_prog_get_info_by_fd(prog_fd, &info, &len); + if (!ASSERT_OK(err, name)) + return; + if (!ASSERT_EQ(info.nr_map_ids, 1, name)) + return; + ASSERT_EQ(map_ids[0], map_id, name); +} + +/* + * libbpf binds the arena of an object to every program in it: a callback + * with an __arena argument loads without referring to the arena, and an + * unrelated program gets the arena as well. + */ +static void arena_autobind(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + int err; + + skel = struct_ops_arena_autobind__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open_and_load")) + return; + + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + assert_prog_map(bpf_program__fd(skel->progs.test_arena_cb), info.id, "test_arena_cb"); + assert_prog_map(bpf_program__fd(skel->progs.unrelated), info.id, "unrelated"); + + attach_and_trigger(skel->maps.testmod_arena_autobind, skel->progs.trigger); +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* + * The same binding through bpf_object__prepare() and bpf_program__clone(), the + * way veristat loads a program: the arena takes slot 0 of a continuous + * fd_array, next to the module BTF of the kfunc the trigger calls. + */ +static void arena_autobind_clone(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + struct bpf_program *prog; + int err, fd; + + skel = struct_ops_arena_autobind__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open")) + return; + + err = bpf_object__prepare(skel->obj); + if (!ASSERT_OK(err, "prepare")) + goto out; + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + bpf_object__for_each_program(prog, skel->obj) { + fd = bpf_program__clone(prog, NULL); + if (!ASSERT_GE(fd, 0, bpf_program__name(prog))) + continue; + assert_prog_map(fd, info.id, bpf_program__name(prog)); + close(fd); + } +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* + * An arena exclusive to a program is bound to that program all the same: the + * kernel checks an exclusive map against the digest of the final program, so + * the binding through the fd_array passes for the program the map is + * exclusive to. + */ +static void arena_autobind_excl(void) +{ + struct struct_ops_arena_autobind *skel; + struct bpf_map_info info = {}; + __u32 len = sizeof(info); + int err; + + skel = struct_ops_arena_autobind__open(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_autobind__open")) + return; + /* the other programs need the arena too, and cannot have it */ + bpf_map__set_autocreate(skel->maps.testmod_arena_autobind, false); + bpf_program__set_autoload(skel->progs.trigger, false); + err = bpf_map__set_exclusive_program(skel->maps.arena, skel->progs.unrelated); + if (!ASSERT_OK(err, "set_exclusive_program")) + goto out; + + err = struct_ops_arena_autobind__load(skel); + if (!ASSERT_OK(err, "load")) + goto out; + err = bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.arena), &info, &len); + if (!ASSERT_OK(err, "arena_info")) + goto out; + assert_prog_map(bpf_program__fd(skel->progs.unrelated), info.id, "unrelated"); +out: + struct_ops_arena_autobind__destroy(skel); +} + +/* The same through the vmlinux.h rendering of the members, which takes casts. */ +static void arena_member_render(void) +{ + struct struct_ops_arena_member_render *skel; + + skel = struct_ops_arena_member_render__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_arena_member_render__open_and_load")) + return; + + attach_and_trigger(skel->maps.testmod_arena_member_render, skel->progs.trigger); + struct_ops_arena_member_render__destroy(skel); +} #endif /* @@ -116,12 +286,37 @@ void serial_test_struct_ops_arena(void) * arena arguments", so the programs cannot even load. */ #if defined(__x86_64__) || defined(__aarch64__) + int has_tag; + if (test__start_subtest("arena_arg")) arena_arg(); if (test__start_subtest("arena_arg_fail")) arena_arg_fail(); if (test__start_subtest("arena_arg_attach")) arena_arg_attach(); + /* the tags reach the module BTF only from a toolchain with btf_type_tag support */ + has_tag = testmod_member_has_type_tag("bpf_testmod_arena_member_args", "ptr", "arena"); + if (ASSERT_GE(has_tag, 0, "testmod_btf") && has_tag) { + if (test__start_subtest("arena_member")) + arena_member(); + if (test__start_subtest("arena_member_render")) + arena_member_render(); + RUN_TESTS(struct_ops_arena_member_fail__union); + RUN_TESTS(struct_ops_arena_member_fail__xchg); + RUN_TESTS(struct_ops_arena_member_fail__mismatch); + RUN_TESTS(struct_ops_arena_member_fail__mismatch_mem); + RUN_TESTS(struct_ops_arena_member_fail__write); + RUN_TESTS(struct_ops_arena_member_fail__no_arena); + } else if (test__start_subtest("arena_member")) { + /* so that the missing member tests show up as skipped */ + test__skip(); + } + if (test__start_subtest("arena_autobind")) + arena_autobind(); + if (test__start_subtest("arena_autobind_clone")) + arena_autobind_clone(); + if (test__start_subtest("arena_autobind_excl")) + arena_autobind_excl(); #else test__skip(); #endif diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c new file mode 100644 index 000000000000..028f070f2e5f --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_nullable.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include + +#include "struct_ops_nullable.skel.h" +#include "struct_ops_nullable_fail.skel.h" +#include "struct_ops_nullable_fail__percpu.skel.h" +#include "struct_ops_nullable_probed__plain.skel.h" +#include "struct_ops_nullable_probed__scalar.skel.h" +#include "struct_ops_nullable_probed__untrusted.skel.h" + +/* + * Attach a callback that gets a trusted and a trusted __nullable task pointer + * through its argument struct and drive it through the + * bpf_testmod_ops3_call_test_nullable() kfunc. + */ +static void nullable_member(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct struct_ops_nullable *skel; + struct bpf_link *link; + int err; + + skel = struct_ops_nullable__open_and_load(); + if (!ASSERT_OK_PTR(skel, "struct_ops_nullable__open_and_load")) + return; + + link = bpf_map__attach_struct_ops(skel->maps.testmod_nullable); + if (!ASSERT_OK_PTR(link, "attach_struct_ops")) + goto out; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.trigger), &topts); + ASSERT_OK(err, "test_run"); + ASSERT_EQ(topts.retval, 0, "trigger_retval"); + + bpf_link__destroy(link); +out: + struct_ops_nullable__destroy(skel); +} + +/* Serialized because it attaches the singleton bpf_testmod_ops3. */ +void serial_test_struct_ops_nullable(void) +{ + int has_tag; + + /* the tag reaches the module BTF only from a toolchain with btf_type_tag support */ + has_tag = testmod_member_has_type_tag("bpf_testmod_nullable_args", "task_or_null", "nullable"); + if (!ASSERT_GE(has_tag, 0, "testmod_btf")) + return; + if (!has_tag) { + printf("%s:SKIP: no nullable type tag in the bpf_testmod BTF\n", __func__); + test__skip(); + return; + } + if (test__start_subtest("nullable_member")) + nullable_member(); + RUN_TESTS(struct_ops_nullable_fail); + RUN_TESTS(struct_ops_nullable_fail__percpu); + RUN_TESTS(struct_ops_nullable_probed__plain); + RUN_TESTS(struct_ops_nullable_probed__scalar); + RUN_TESTS(struct_ops_nullable_probed__untrusted); +} diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index c69c0f7e9cc8..6df440d82675 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -3,6 +3,7 @@ #include #include "arena_kfunc.skel.h" +#include "arena_noarena.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" #include "cb_refs.skel.h" @@ -193,6 +194,7 @@ static void run_tests_aux(const char *skel_name, } while (0) void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } +void test_arena_noarena(void) { RUN_TESTS(arena_noarena); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index 96f7aacf4d9f..5d1cb1a7e8ee 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -192,14 +192,55 @@ int arena_arg_unpopulated(void *ctx) return 0; } +/* + * A subprogram whose only arena use is the argument of a kfunc needs the + * arena base too, the JIT rebases the argument on it. The address arrives + * as a number, so no cast marks the subprogram either. + */ +__noinline int arena_arg_from_subprog(unsigned long addr) +{ + return bpf_kfunc_arena_arg_test((u64 *)addr) != 41; +} + +/* + * A NULL constant for the __arena argument takes an early exit in the + * verifier; the call needs the arena base all the same, the JIT rebases + * the argument with a NULL test. + */ SEC("syscall") __arch_x86_64 __arch_arm64 -__failure __msg("arena pointer requires a program with an associated arena") -int arena_arg_no_arena(void *ctx) +__success __retval(0) +int arena_arg_null(void *ctx) { - bpf_kfunc_arena_arg_test((u64 *)1); + return bpf_kfunc_arena_cap_nullable_test(NULL) != 0; +} + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__success __retval(0) +int arena_arg_subprog(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + if (arena_arg_from_subprog((unsigned long)val)) + ret = 2; + else + ret = *val != 42 ? 3 : 0; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); + return ret; +#else return 0; +#endif } SEC("syscall") diff --git a/tools/testing/selftests/bpf/progs/arena_noarena.c b/tools/testing/selftests/bpf/progs/arena_noarena.c new file mode 100644 index 000000000000..edbd40cd5be8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_noarena.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Programs without an arena. They cannot share an object with an arena map, + * libbpf binds the arena of an object to every program in it. + */ + +SEC("syscall") +__arch_x86_64 +__arch_arm64 +__failure __msg("arena pointer requires a program with an associated arena") +int arena_arg_no_arena(void *ctx) +{ + bpf_kfunc_arena_arg_test((u64 *)1); + return 0; +} + +SEC("syscall") +__flag(BPF_F_ARENA_SCALAR) +__arch_x86_64 +__arch_arm64 +__description("arena_scalar: no arena, no access through a number") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void no_arena(void) +{ + asm volatile (" \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" ::: __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/map_excl.c b/tools/testing/selftests/bpf/progs/map_excl.c index d461684728e4..90f127073dff 100644 --- a/tools/testing/selftests/bpf/progs/map_excl.c +++ b/tools/testing/selftests/bpf/progs/map_excl.c @@ -27,7 +27,8 @@ int should_have_access(void *ctx) SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") int should_not_have_access(void *ctx) { - int key = 0, value = 0xdeadbeef; + /* a different value, so that the two programs do not share a digest */ + int key = 0, value = 0xbaadf00d; bpf_map_update_elem(&excl_map, &key, &value, 0); return 0; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c new file mode 100644 index 000000000000..495faa0f4a8b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_autobind.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include "bpf_experimental.h" +#include +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * Neither the callback nor the unrelated program refers to the arena. libbpf + * binds it to them all the same, so the callback can use the __arena argument + * the kernel hands it. + */ +SEC("struct_ops/test_arena") +int test_arena_cb(unsigned long long *ctx) +{ + u64 __arena *ptr = (u64 __arena *)ctx[0]; + + *ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_autobind = { + .test_arena = (void *)test_arena_cb, +}; + +SEC("syscall") +int unrelated(void *ctx) +{ + return 0; +} + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + if (bpf_testmod_ops3_call_test_arena((u64 *)val)) + return 2; + if (*val != 42) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c new file mode 100644 index 000000000000..a2218cd81619 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include +#include "bpf_experimental.h" +#include +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + /* page 0 hosts the arena globals, page 1 is for allocations */ + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callback with the arena */ +u64 __arena arena_touch; +/* the register value a load of args->ptr produced in the callback */ +u64 __arena member_ptr_val; + +/* + * The header is shared with the module, so the members are u64 __arena * on + * this side too and the loads need no cast. The verifier derives the type of + * the loaded value from the module BTF, where the members carry the tag. + */ +SEC("struct_ops/test_arena_member") +int BPF_PROG(test_arena_member_cb, struct bpf_testmod_arena_member_args *args) +{ + u64 __arena *ptr = args->ptr, *acq; + u64 raw; + + arena_touch++; + + /* the register right after the load, untouched by the compiler */ + asm volatile("%[raw] = *(u64 *)(%[args] + %[off])" + : [raw]"=r"(raw) + : [args]"r"(args), + [off]"i"(offsetof(struct bpf_testmod_arena_member_args, ptr))); + member_ptr_val = raw; + + *ptr += 1; + /* an acquire load of the member yields the same arena pointer */ + asm volatile("r1 = %[args];\n" + ".8byte 0x00000100000010db;\n" /* r0 = load_acquire((u64 *)(r1 + 0)) */ + "%[acq] = r0;" + : [acq]"=r"(acq) + : [args]"r"(args) + : "r0", "r1"); + if (acq != ptr) + return 0xbad; + *acq += 1; + if (args->null_ptr) + return 0xbad; + *args->nested.ptr += 1; + if (args->cookie != 0xc00c1e) + return 0xbad; + /* + * @faulting is a plain kernel pointer, so the member behind it is read + * with a probed load, which the JIT skips for a user address or which + * faults, and reads as NULL rather than as an arena pointer. + */ + if (args->faulting->ptr) + return 0xbad; + /* an arena pointer to kfuncs as well, the kfunc adds one */ + bpf_kfunc_arena_arg_test((u64 *)args->ptr); + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member = { + .test_arena_member = (void *)test_arena_member_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena_member((u64 *)val); + if (ret) + return 2; + /* ptr, its acquire load, nested.ptr and the kfunc each added one */ + if (*val != 45) + return 3; + + /* + * The callback saw (u32)(kaddr - kern_vm_start), the arena offset of + * val with the upper 32 bits clear, the same as an __arena argument. + */ + if (member_ptr_val != (u32)(u64)val) + return 4; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c new file mode 100644 index 000000000000..0b4147894122 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The same load reaches args->ptr on one path and args->cookie on the other, + * the branch on the unknown cookie value makes the verifier walk both. The + * rewrite is per insn, so the paths have to agree on what they load. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("loads an arena pointer member on one path and something else on another") +__naked int mismatch_member(void) +{ + asm volatile ( + "r6 = *(u64 *)(r1 + 0);" + "r2 = %[arena] ll;" + "r0 = *(u64 *)(r6 + %[cookie_off]);" + "if r0 == 0 goto 1f;" + "r6 += %[cookie_off];" +"1: r0 = *(u64 *)(r6 + 0);" + "r0 = 0;" + "exit;" + : + : __imm_addr(arena), + __imm_const(cookie_off, offsetof(struct bpf_testmod_arena_member_args, cookie)) + : __clobber_all); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)mismatch_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c new file mode 100644 index 000000000000..ff6d4db47cfd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__mismatch_mem.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The same load reads args->ptr on one path and the memory argument on the + * other. The verifier lets a load insn serve a kernel struct pointer and a + * memory pointer, see save_aux_ptr_type(), but the rewrite of the arena + * member would corrupt the value the other path loads. A global function + * is the only place a struct_ops program of this type gets a memory pointer. + * The loaded value goes nowhere: an ALU insn on it would be an arena + * pointer on one path and a scalar on the other, which the verifier rejects + * on its own before the fixup pass gets to the load. + */ +__noinline int load_either(unsigned long long *ctx __arg_ctx, u64 *mem __arg_nonnull) +{ + struct bpf_testmod_arena_member_args *args = (void *)ctx[0]; + void *p = args; + + if (args->cookie == 0) + p = mem; + /* one load insn, reached with both pointers */ + asm volatile("r0 = *(u64 *)(%[p] + 0)" :: [p]"r"(p) : "r0"); + return 0; +} + +SEC("struct_ops/test_arena_member") +__failure __msg("loads an arena pointer member on one path and something else on another") +int BPF_PROG(mismatch_mem, struct bpf_testmod_arena_member_args *args) +{ + u64 local = 0; + + asm volatile("" :: "r"(&arena)); + return load_either(ctx, &local); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)mismatch_mem, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c new file mode 100644 index 000000000000..cdd8ea740478 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__no_arena.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* No arena in the object: the member cannot be translated into anything. */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of struct bpf_testmod_arena_member_args " + "requires a program with an associated arena") +int BPF_PROG(no_arena_member, struct bpf_testmod_arena_member_args *args) +{ + return args->ptr != NULL; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)no_arena_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c new file mode 100644 index 000000000000..d45666fd482c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__union.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * An offset into a union resolves to its first member, so a program reading + * raw there would get an arena pointer it did not ask for. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of bpf_testmod_arena_union is inside a union") +int BPF_PROG(union_member, struct bpf_testmod_arena_member_args *args) +{ + asm volatile("" :: "r"(&arena)); + return *args->u.ptr != 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)union_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c new file mode 100644 index 000000000000..ff0761980948 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__write.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* + * The kernel owns the member, a program only reads it. A struct_ops program + * cannot store through its arguments at all, so the store is refused before + * the tag comes into play; the exchange test reaches the arena member check. + */ +SEC("struct_ops/test_arena_member") +__failure __msg("only read is supported") +int BPF_PROG(write_member, struct bpf_testmod_arena_member_args *args) +{ + asm volatile("" :: "r"(&arena)); + args->ptr = NULL; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)write_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c new file mode 100644 index 000000000000..0d814166cabc --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_fail__xchg.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + +/* An atomic that writes the member is neither a plain nor an acquire load. */ +SEC("struct_ops/test_arena_member") +__failure __msg("arena pointer member ptr of struct " + "bpf_testmod_arena_member_args can only be read with a plain or an acquire load") +__naked int xchg_member(void) +{ + asm volatile ( + "r1 = *(u64 *)(r1 + 0);" + "r2 = %[arena] ll;" + "r0 = 0;" + /* r0 = xchg_64((u64 *)(r1 + 0), r0): BPF_STX | BPF_ATOMIC | BPF_DW, imm BPF_XCHG */ + ".8byte 0x000000e1000001db;" + "r0 = 0;" + "exit;" + : + : __imm_addr(arena) + : __clobber_all); +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_fail = { + .test_arena_member = (void *)xchg_member, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c new file mode 100644 index 000000000000..09b4c740bdca --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_arena_member_render.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include +#include "bpf_experimental.h" +#include + +char _license[] SEC("license") = "GPL"; + +/* + * The module types as a vmlinux.h generated by a libbpf that does not render + * the arena tag with its address space shows them: an attribute on a pointer + * in the kernel address space, so the program has to cast the member to the + * arena address space before using it. + */ +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#define __nullable_tag __attribute__((btf_type_tag("nullable"))) + +struct bpf_testmod_arena_member_args { + __u64 __arena_tag *ptr; + __u64 __arena_tag __nullable_tag *null_ptr; + struct { + __u64 __arena_tag *ptr; + } nested; +} __attribute__((preserve_access_index)); + +struct bpf_testmod_ops3 { + int (*test_arena_member)(struct bpf_testmod_arena_member_args *args); +}; + +int bpf_testmod_ops3_call_test_arena_member(__u64 *ptr__arena) __ksym; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +/* also associates the callback with the arena */ +u64 __arena arena_touch; + +SEC("struct_ops/test_arena_member") +int BPF_PROG(test_arena_member_render_cb, struct bpf_testmod_arena_member_args *args) +{ + u64 __arena *ptr = (u64 __arena *)args->ptr; + + arena_touch++; + *ptr += 1; + if (args->null_ptr) + return 0xbad; + *(u64 __arena *)args->nested.ptr += 1; + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_arena_member_render = { + .test_arena_member = (void *)test_arena_member_render_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) + u64 __arena *val; + int ret; + + val = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + if (!val) + return 1; + + *val = 41; + ret = bpf_testmod_ops3_call_test_arena_member((u64 *)val); + if (ret) + return 2; + /* ptr and nested.ptr each added one */ + if (*val != 43) + return 3; + + bpf_arena_free_pages(&arena, (void __arena *)val, 1); +#endif + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable.c new file mode 100644 index 000000000000..929969e1e012 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "../test_kmods/bpf_testmod.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +char _license[] SEC("license") = "GPL"; + +/* an untrusted pointer to the argument struct, never set, for the verifier */ +struct untrusted_val { + struct bpf_testmod_nullable_args __kptr_untrusted *args; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct untrusted_val); +} untrusted_map SEC(".maps"); + +SEC("struct_ops/test_nullable") +int BPF_PROG(test_nullable_cb, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_nullable_args *u; + struct untrusted_val *v; + struct task_struct *t; + int ret = 0, *i, zero = 0; + + /* trusted, so good for a kfunc as is */ + t = bpf_task_acquire(args->task); + if (!t) + return -1; + bpf_task_release(t); + + /* trusted but __nullable, so a check has to come first */ + if (args->task_or_null) { + t = bpf_task_acquire(args->task_or_null); + if (!t) + return -1; + bpf_task_release(t); + ret = 1; + } + /* per-CPU and __nullable: a check has to come first, per_cpu_ptr() has none */ + if (args->pcpu_or_null) { + struct bpf_testmod_pcpu *v = bpf_this_cpu_ptr(args->pcpu_or_null); + + if (v->val != 0x5ca1ab1e) + return -1; + } + /* through a plain pointer the loads are probed, __nullable still asks for a check */ + t = args->self->task_or_null; + if (t && t->pid != args->task->pid) + return -1; + /* a pointer to a scalar walks as untrusted memory, __nullable applies all the same */ + i = args->int_or_null; + if (i && *i != 7) + return -1; + /* through an untrusted pointer the member is probed too, and stays nullable */ + v = bpf_map_lookup_elem(&untrusted_map, &zero); + u = v ? v->args : NULL; + if (u) { + t = u->task_or_null; + if (t && t->pid != args->task->pid) + return -1; + } + return ret; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable = { + .test_nullable = (void *)test_nullable_cb, +}; + +SEC("syscall") +int trigger(void *ctx) +{ + /* one call without task_or_null, one with */ + return bpf_testmod_ops3_call_test_nullable() != 2; +} diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c new file mode 100644 index 000000000000..c15b3a8c8d39 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* The member is trusted, but __nullable, so it needs a check before the kfunc. */ +SEC("struct_ops/test_nullable") +__failure __msg("Possibly NULL pointer passed to trusted") +int BPF_PROG(unchecked_nullable, struct bpf_testmod_nullable_args *args) +{ + struct task_struct *t; + + t = bpf_task_acquire(args->task_or_null); + if (t) + bpf_task_release(t); + return 0; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_fail = { + .test_nullable = (void *)unchecked_nullable, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c new file mode 100644 index 000000000000..652015189162 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_fail__percpu.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * The member is per-CPU and __nullable: per_cpu_ptr() shifts NULL like any + * other pointer, so a check has to come before the helper. + */ +SEC("struct_ops/test_nullable") +__failure __msg("Possibly NULL pointer passed to trusted R1") +int BPF_PROG(unchecked_percpu, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_pcpu *v = bpf_this_cpu_ptr(args->pcpu_or_null); + + return v->val; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_fail = { + .test_nullable = (void *)unchecked_percpu, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c new file mode 100644 index 000000000000..c45f6d34f7d2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__plain.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * Through a plain pointer the member is probed, which tolerates NULL, so the + * tag asks for no check there, as the BTF_TYPE_SAFE_*_OR_NULL lists never did. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_plain, struct bpf_testmod_nullable_args *args) +{ + return args->self->task_or_null->pid; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_plain, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c new file mode 100644 index 000000000000..ff768746b0da --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__scalar.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +/* + * A pointer to a scalar walks as untrusted memory, which is probed, so the + * tag asks for no check there. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_scalar, struct bpf_testmod_nullable_args *args) +{ + return *args->int_or_null; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_scalar, +}; diff --git a/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c new file mode 100644 index 000000000000..961999139841 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/struct_ops_nullable_probed__untrusted.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod.h" + +char _license[] SEC("license") = "GPL"; + +struct untrusted_val { + struct bpf_testmod_nullable_args __kptr_untrusted *args; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct untrusted_val); +} untrusted_map SEC(".maps"); + +/* + * Through an untrusted pointer the member is probed, which tolerates NULL, so + * the tag asks for no check there. + */ +SEC("struct_ops/test_nullable") +__success +int BPF_PROG(unchecked_untrusted, struct bpf_testmod_nullable_args *args) +{ + struct bpf_testmod_nullable_args *u; + struct untrusted_val *v; + int zero = 0; + + v = bpf_map_lookup_elem(&untrusted_map, &zero); + u = v ? v->args : NULL; + if (!u) + return 0; + return u->task_or_null->pid; +} + +SEC(".struct_ops.link") +struct bpf_testmod_ops3 testmod_nullable_probed = { + .test_nullable = (void *)unchecked_untrusted, +}; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c index ecfc6873ffa9..6f0efd9202ca 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c @@ -991,19 +991,6 @@ __naked void no_flag(void) : __clobber_all); } -SEC("syscall") -__arena_scalar -__description("arena_scalar: no arena, no access through a number") -__failure __msg("R6 invalid mem access 'scalar'") -__naked void no_arena(void) -{ - asm volatile (" \ - r6 = 0x100000000000 ll; \ - r0 = *(u64 *)(r6 + 0); \ - exit; \ -" ::: __clobber_all); -} - SEC("syscall") __arena_scalar __description("arena_scalar: pointer that is NULL is an address in arena") diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 613abd27be5d..6380f52f4dd2 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -506,6 +506,16 @@ static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair return 0; } +static int bpf_testmod_ops3__test_arena_member(struct bpf_testmod_arena_member_args *args) +{ + return 0; +} + +static int bpf_testmod_ops3__test_nullable(struct bpf_testmod_nullable_args *args) +{ + return 0; +} + static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_1 = bpf_testmod_test_3, .test_2 = bpf_testmod_test_4, @@ -513,6 +523,8 @@ static struct bpf_testmod_ops3 __bpf_testmod_ops3 = { .test_arena_nullable = bpf_testmod_ops3__test_arena_nullable, .test_arena_stack = bpf_testmod_ops3__test_arena_stack, .test_arena_multislot = bpf_testmod_ops3__test_arena_multislot, + .test_arena_member = bpf_testmod_ops3__test_arena_member, + .test_nullable = bpf_testmod_ops3__test_nullable, }; static void bpf_testmod_test_struct_ops3(void) @@ -553,6 +565,58 @@ __bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena) return st_ops3->test_arena_multislot(p, ptr__arena); } +/* The callback gets the kernel address of @ptr__arena behind each __arena member. */ +__bpf_kfunc int bpf_testmod_ops3_call_test_arena_member(u64 *ptr__arena) +{ + struct bpf_testmod_arena_member_args args = { + .ptr = ptr__arena, + .nested.ptr = ptr__arena, + .u.ptr = ptr__arena, + .cookie = 0xc00c1e, + /* in the first page, which is never mapped */ + .faulting = (void *)16, + }; + + if (!st_ops3 || !st_ops3->test_arena_member) + return -EOPNOTSUPP; + return st_ops3->test_arena_member(&args); +} + +/* + * The members of struct bpf_testmod_nullable_args are trusted the way the + * verifier's BTF_TYPE_SAFE_TRUSTED() lists vmlinux members: through a struct + * of the same name with the __safe_trusted suffix in the BTF of the module. + */ +struct bpf_testmod_nullable_args__safe_trusted { + struct task_struct *task; + struct task_struct *task_or_null; +}; + +static DEFINE_PER_CPU(struct bpf_testmod_pcpu, bpf_testmod_nullable_pcpu); + +/* + * Call test_nullable() with the __nullable members unset, then set. The + * callback returns 1 when it found a task in @task_or_null, so the expected + * result is 2. + */ +__bpf_kfunc int bpf_testmod_ops3_call_test_nullable(void) +{ + struct bpf_testmod_nullable_args args = { .task = current }; + int seven = 7, ret; + + BTF_TYPE_EMIT(struct bpf_testmod_nullable_args__safe_trusted); + + if (!st_ops3 || !st_ops3->test_nullable) + return -EOPNOTSUPP; + args.self = &args; + ret = st_ops3->test_nullable(&args); + args.task_or_null = current; + this_cpu_write(bpf_testmod_nullable_pcpu.val, 0x5ca1ab1e); + args.pcpu_or_null = &bpf_testmod_nullable_pcpu; + args.int_or_null = &seven; + return ret | st_ops3->test_nullable(&args) << 1; +} + struct bpf_testmod_btf_type_tag_1 { int a; }; @@ -980,6 +1044,8 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_member) +BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_nullable) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_kasan_poison) @@ -1956,6 +2022,7 @@ static const struct bpf_verifier_ops bpf_testmod_verifier_ops = { }; static const struct bpf_verifier_ops bpf_testmod_verifier_ops3 = { + .get_func_proto = bpf_base_func_proto, .is_valid_access = bpf_testmod_ops_is_valid_access, }; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 2b12f81bf748..4ba9def74404 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -114,6 +114,76 @@ struct bpf_testmod_arena_pair { u64 b; }; +/* + * The qualifiers are BTF type tags for the verifier. The module gets __arena + * and __nullable from bpf_compiler_types.h. A BPF program gets __arena from + * bpf_arena_common.h, and the same definition here when it includes this + * header first, and needs __nullable and __percpu empty. User space needs all + * of them empty. + */ +#ifdef __KERNEL__ +#include +#endif +#if defined(__BPF__) && !defined(__arena) +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM) +#define __arena __attribute__((address_space(1))) __attribute__((btf_type_tag("arena"))) +#else +#define __arena __attribute__((btf_type_tag("arena"))) +#endif +#endif +#ifndef __arena +#define __arena +#endif +#ifndef __nullable +#define __nullable +#endif +#ifndef __percpu +#define __percpu +#endif + +struct bpf_testmod_pcpu { + int val; +}; + +struct bpf_testmod_arena_nested { + u64 __arena *ptr; +}; + +union bpf_testmod_arena_union { + u64 __arena *ptr; + u64 raw; +}; + +/* Arguments of test_arena_member(), the pointers hold kernel addresses of arena memory */ +struct bpf_testmod_arena_member_args { + /* the value the caller passed, never NULL */ + u64 __arena *ptr; + /* always NULL */ + u64 __arena __nullable *null_ptr; + /* the same value again, behind a nested struct */ + struct bpf_testmod_arena_nested nested; + /* the same value again, but a load from a union is rejected */ + union bpf_testmod_arena_union u; + /* a scalar, loaded by the same insn as @ptr in a verifier test */ + u64 cookie; + /* a bad pointer to a struct like this one, a load through it faults */ + struct bpf_testmod_arena_member_args *faulting; +}; + +/* Arguments of test_nullable(), trusted through the __safe_trusted alias in the module */ +struct bpf_testmod_nullable_args { + /* never NULL */ + struct task_struct *task; + /* NULL on the first call, set on the second */ + struct task_struct __nullable *task_or_null; + /* NULL on the first call, a per-CPU variable on the second */ + struct bpf_testmod_pcpu __percpu __nullable *pcpu_or_null; + /* NULL on the first call, set on the second */ + int __nullable *int_or_null; + /* this struct again, through a plain pointer, which the verifier probes */ + struct bpf_testmod_nullable_args *self; +}; + struct bpf_testmod_ops3 { int (*test_1)(void); int (*test_2)(void); @@ -125,6 +195,10 @@ struct bpf_testmod_ops3 { u64 g, u64 h, u64 *ptr); /* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */ int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr); + /* Used to test __arena pointer members. */ + int (*test_arena_member)(struct bpf_testmod_arena_member_args *args); + /* Used to test __nullable pointer members. */ + int (*test_nullable)(struct bpf_testmod_nullable_args *args); }; struct st_ops_args { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 8f2f8b8a26e3..3b810f1e438f 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -224,6 +224,8 @@ int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym; int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym; int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_arena_member(__u64 *ptr__arena) __ksym; +int bpf_testmod_ops3_call_test_nullable(void) __ksym; __u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b, __u32 c, __u64 d) __ksym; diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index 202c40e536b2..1e3e31ac36eb 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "disasm.h" #include "test_progs.h" @@ -644,3 +645,47 @@ int compare_text_to_expected(const char *actual, const char *expected) } return -1; } + +int testmod_member_has_type_tag(const char *struct_name, const char *member, const char *tag) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_member *m; + const struct btf_type *t; + int err = 0, id, i; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return -errno; + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) { + err = -errno; + goto out; + } + id = btf__find_by_name_kind_own(module_btf, struct_name, BTF_KIND_STRUCT); + if (id < 0) { + err = id; + goto out; + } + t = btf__type_by_id(module_btf, id); + err = -ENOENT; + for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { + if (strcmp(btf__name_by_offset(module_btf, m->name_off), member)) + continue; + err = 0; + /* the tag sits on the pointee, behind the pointer and other modifiers */ + for (id = m->type; id; id = t->type) { + t = btf__type_by_id(module_btf, id); + if (btf_is_type_tag(t) && !strcmp(btf__name_by_offset(module_btf, t->name_off), tag)) { + err = 1; + break; + } + if (!btf_is_ptr(t) && !btf_is_mod(t) && !btf_is_typedef(t)) + break; + } + break; + } +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + return err; +} diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index d5083fb92ccd..36a75158495b 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -67,4 +67,11 @@ int stack_mprotect(void); /* Runs diff(1) on mismatch */ int compare_text_to_expected(const char *actual, const char *expected); +/* + * Whether @member of struct @struct_name in the BTF of bpf_testmod carries + * the BTF type tag @tag: 1 if so, 0 if not, negative on a BTF error or when + * the struct or the member is missing. + */ +int testmod_member_has_type_tag(const char *struct_name, const char *member, const char *tag); + #endif /* __TESTING_HELPERS_H */