diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 951a90458d18..54481da4ed2d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -793,6 +793,14 @@ struct bpf_insn_aux_data { */ u64 fastcall_spills_num:3; u64 arg_prog:4; + /* + * What a load found, on any path: an __arena pointer member of a + * kernel struct, which arena_member_fixup() rewrites, one that is + * __nullable as well, or anything else through any pointer. + */ + u64 arena_member:1; + u64 arena_member_nullable:1; + u64 non_arena_load:1; /* below flags are initialized once */ u64 jmp_point:1; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 012b6c5c3eff..538129ed2418 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7707,6 +7707,7 @@ enum bpf_struct_walk_result { WALK_SCALAR = 0, WALK_PTR, WALK_PTR_UNTRUSTED, + WALK_PTR_ARENA, WALK_STRUCT, }; @@ -7987,6 +7988,30 @@ static int btf_struct_walk(struct bpf_verifier_log *log, const struct btf *btf, if (btf_ptr_has_type_tag(btf, mtype, "nullable")) tmp_flag |= PTR_MAYBE_NULL; + /* + * An __arena member of a kernel struct holds the kernel + * address of arena memory, which the verifier turns into + * an arena pointer on load. Program BTF is not affected: + * there the tag is what the compiler emits for the arena + * address space. An offset into a union resolves to its + * first member, so a program reading another member there + * would get an arena pointer it did not ask for; the walk + * marked such a union untrusted. + */ + if (btf_is_kernel(btf) && btf_type_is_arena_ptr(btf, mtype)) { + if (*flag & PTR_UNTRUSTED) { + bpf_log(log, + "arena pointer member %s of %s is inside a union\n", + mname, tname); + return -EACCES; + } + *next_btf_id = id; + *flag |= tmp_flag & PTR_MAYBE_NULL; + if (field_name) + *field_name = mname; + return WALK_PTR_ARENA; + } + if (btf_type_is_struct(stype)) { *next_btf_id = id; *flag |= tmp_flag; @@ -8089,6 +8114,11 @@ int btf_struct_access(struct bpf_verifier_log *log, *flag = MEM_RDONLY | PTR_UNTRUSTED; ret = PTR_TO_MEM; goto check_variable_offset; + case WALK_PTR_ARENA: + *next_btf_id = id; + *flag = tmp_flag; + ret = PTR_TO_ARENA; + goto check_variable_offset; case WALK_SCALAR: ret = SCALAR_VALUE; goto check_variable_offset; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index bfbb0cf5c0a8..34b53ebb5841 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -882,6 +882,37 @@ static bool ext_prog_takes_prologue(const struct bpf_verifier_env *env) * struct __sk_buff -> struct sk_buff * struct bpf_sock_ops -> struct sock */ +/* + * A load through an __arena pointer member of a kernel struct reads the kernel + * address of arena memory. Append the conversion to the arena pointer form, + * (u32)(kaddr - kern_vm_start), that the struct_ops trampoline applies to + * __arena arguments. A __nullable member keeps NULL as NULL, tested on the + * full kernel address, and so does a probed load, which reads as 0 when it + * faults. Returns the number of insns in @buf, 0 when the insn loads no such + * member, or an error when it does so on some paths only. + */ +static int arena_member_fixup(struct bpf_verifier_env *env, int idx, struct bpf_insn *insn, + struct bpf_insn *buf) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[idx]; + int cnt = 0; + u32 base; + + if (!aux->arena_member) + return 0; + if (aux->non_arena_load) { + verbose(env, "an insn loads an arena pointer member on one path and something else on another\n"); + return -EINVAL; + } + + base = bpf_arena_get_kern_vm_start(env->prog->aux->arena); + buf[cnt++] = *insn; + if (aux->arena_member_nullable || BPF_MODE(insn->code) == BPF_PROBE_MEM) + buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, insn->dst_reg, 0, 1); + buf[cnt++] = BPF_ALU32_IMM(BPF_SUB, insn->dst_reg, (s32)base); + return cnt; +} + int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) { struct bpf_subprog_info *subprogs = env->subprog_info; @@ -1026,6 +1057,21 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; continue; + } else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && + insn->imm == BPF_LOAD_ACQ && + env->insn_aux_data[i + delta].arena_member) { + /* an acquire load of an __arena member is rewritten like a plain load */ + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; + continue; } else if (insn->code == (BPF_JMP | BPF_EXIT) && epilogue_cnt && i + delta < subprogs[1].start) { @@ -1117,11 +1163,23 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn->code = BPF_LDX | BPF_PROBE_MEMSX | BPF_SIZE(insn->code); env->prog->aux->num_exentries++; - continue; - } - if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, - "access to a fault prone pointer is not rewritten as a probed one")) + } else if (verifier_bug_if(bpf_may_fault_on_deref(ptr_type), env, + "access to a fault prone pointer is not rewritten as a probed one")) { return -EFAULT; + } + + cnt = arena_member_fixup(env, i + delta, insn, insn_buf); + if (cnt < 0) + return cnt; + if (!cnt) + continue; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = new_prog; + insn = new_prog->insnsi + i + delta; continue; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 1679ffec431b..015dc1bccebe 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6585,6 +6585,45 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env, "__safe_trusted_or_null"); } +/* + * The member holds the kernel address of arena memory, which the rewritten + * load turns into an arena pointer, see arena_member_fixup(). NULL needs no + * tracking, PTR_TO_ARENA covers it. + */ +static int check_arena_member_load(struct bpf_verifier_env *env, struct bpf_reg_state *regs, + int value_regno, enum bpf_type_flag flag, const char *tname, + const char *field_name) +{ + struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx]; + struct bpf_insn *insn; + + /* a helper reading the member as bytes needs no rewrite */ + if (value_regno < 0) + return 0; + if (!env->prog->aux->arena) { + verbose(env, + "arena pointer member %s of struct %s requires a program with an associated arena\n", + field_name, tname); + return -EINVAL; + } + /* only a plain load or an acquire load is rewritten */ + insn = &env->prog->insnsi[env->insn_idx]; + if (BPF_CLASS(insn->code) != BPF_LDX && + !(insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) && insn->imm == BPF_LOAD_ACQ)) { + verbose(env, + "arena pointer member %s of struct %s can only be read with a plain or an acquire load\n", + field_name, tname); + return -EACCES; + } + + aux->arena_member = true; + if (flag & PTR_MAYBE_NULL) + aux->arena_member_nullable = true; + mark_reg_unknown(env, regs, value_regno); + regs[value_regno].type = PTR_TO_ARENA; + return 0; +} + static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, struct bpf_reg_state *reg, argno_t argno, int off, int size, @@ -6694,6 +6733,9 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, if (ret < 0) return ret; + if (ret == PTR_TO_ARENA) + return check_arena_member_load(env, regs, value_regno, flag, tname, field_name); + if (ret != PTR_TO_BTF_ID) { /* just mark; */ @@ -7475,6 +7517,17 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); + /* + * Paths through a kernel struct pointer and through other pointers + * may share the insn, see save_aux_ptr_type(). Record a load that did + * not produce an arena pointer from an __arena member, the only load + * through a kernel struct pointer that does, next to the one + * check_arena_member_load() records, so that arena_member_fixup() can + * tell when the paths disagree. + */ + if (!err && !(base_type(src_reg_type) == PTR_TO_BTF_ID && + regs[insn->dst_reg].type == PTR_TO_ARENA)) + env->insn_aux_data[env->insn_idx].non_arena_load = true; err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx);