diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index f393c3c3d3b4..c27663cc6cc5 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -164,21 +164,22 @@ suffix should be used. 2.3.3 __uninit Annotation ------------------------- -Use ``__uninit`` on a pointer parameter for an output that the kfunc -initializes without reading its incoming contents. +Use ``__uninit`` on a pointer parameter for an output buffer whose incoming +contents the kfunc does not read. -For generic memory buffers, the kfunc must initialize every byte in the -declared range on every return path, including error returns and struct -padding. The range is determined by the pointed-to type or the associated -``__sz`` or ``__szk`` size argument. +For generic memory buffers, the kfunc may leave bytes untouched, including +on error returns. The writable range is determined by the pointed-to type +or the associated ``__sz`` or ``__szk`` size argument. The annotation does not change the accepted pointer types. A stack-backed struct passed as a generic memory buffer must still be scalar-only. -A stack buffer with a verifier-known constant offset and size may be -uninitialized before the call and is considered initialized afterwards. -For variable offsets or sizes, the usual stack-initialization and -variable-offset restrictions still apply. +Generic stack buffers may be uninitialized before the call. The call does +not make previously uninitialized bytes readable unless the program is +allowed to read uninitialized stack memory (normally requiring +``CAP_PERFMON``). Other callers must initialize those bytes themselves +before reading them. Stack bounds and variable-offset restrictions still +apply. For dynptr parameters, ``__uninit`` indicates that the kfunc constructs a dynptr in the supplied storage. For example:: diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 5033b934ffd9..fd22db8bc6c5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -781,7 +781,10 @@ enum bpf_type_flag { */ PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), - /* MEM can be uninitialized. */ + /* + * MEM can be uninitialized. Generic memory outputs need not be fully + * initialized by the callee. + */ MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), /* DYNPTR points to memory local to the bpf program. */ diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 9ddbb20ec1e9..92f528c45605 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1547,12 +1547,14 @@ struct ref_obj_desc { }; /* - * Memory arguments a call fills in, indexed by argument slot. The verifier allows the - * stack to be uninitialized if the range is a known constant. Stack slots are marked as - * STACK_MISC by check_mem_access() after all arguments have been checked. + * Generic MEM_UNINIT arguments, indexed by ABI slot. var_size_mask excludes + * variable-sized buffers from raw mode without losing the output annotation. + * size records constant ranges to mark initialized after checking all arguments, + * only when the caller is allowed to read uninitialized stack memory. */ struct arg_raw_mem_desc { u16 mask; + u16 var_size_mask; int size[MAX_BPF_FUNC_ARGS]; }; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index 696b27383974..1cb5e6a6ffc1 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -2491,7 +2491,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE, .arg4_type = ARG_ANYTHING, }; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 82402d97ce67..501c7ce35cba 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1130,7 +1130,7 @@ const struct bpf_func_proto bpf_snprintf_proto = { .func = bpf_snprintf, .gpl_only = true, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_PTR_TO_CONST_STR, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c94f1214cc3..7f1cc115456f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6993,10 +6993,11 @@ static int check_stack_range_initialized( * but BTF based global subprog validation isn't accurate enough. */ bool allow_poison = access_size < 0 || clobber; - /* The call will initialize the memory; uninitialized stack allowed */ u32 arg_slot = arg_slot_from_argno(argno); - bool raw_mode = meta && arg_slot < MAX_BPF_FUNC_ARGS && - (meta->arg_raw_mem.mask & BIT(arg_slot)); + bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS && + (meta->arg_raw_mem.mask & BIT(arg_slot)); + bool raw_mode = uninit && env->allow_uninit_stack && + !(meta->arg_raw_mem.var_size_mask & BIT(arg_slot)); access_size = abs(access_size); @@ -7035,6 +7036,7 @@ static int check_stack_range_initialized( max_off = reg_smax(reg) + off; } + /* Unprivileged outputs retain each byte's initialization state. */ if (raw_mode) { meta->arg_raw_mem.size[arg_slot] = access_size; return 0; @@ -7054,8 +7056,8 @@ static int check_stack_range_initialized( if (*stype == STACK_MISC) goto mark; if ((*stype == STACK_ZERO) || - (*stype == STACK_INVALID && env->allow_uninit_stack)) { - if (clobber) { + (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) { + if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) { /* helper can write anything into the stack */ *stype = STACK_MISC; } @@ -7074,8 +7076,11 @@ static int check_stack_range_initialized( } if (*stype == STACK_POISON) { - if (allow_poison) + if (allow_poison) { + if (uninit && env->allow_uninit_stack) + *stype = STACK_MISC; goto mark; + } verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", reg_arg_name(env, argno), min_off, i - min_off, access_size); } else if (tnum_is_const(reg->var_off)) { @@ -7224,12 +7229,12 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, meta->msize_max_value = reg_umax(size_reg); /* - * A variable size does not guarantee that the call initializes the whole - * checked range. Disable raw mode for this output and apply the ordinary - * stack initialization checks, including their privilege exceptions. + * Check variable ranges byte by byte instead of using raw mode. Keep the + * MEM_UNINIT annotation so invalid bytes are accepted without marking them + * initialized when the caller cannot read uninitialized stack memory. */ if (!tnum_is_const(size_reg->var_off)) - meta->arg_raw_mem.mask &= ~BIT(arg_slot_from_argno(mem_argno)); + meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno)); if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", @@ -13730,12 +13735,20 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; const struct bpf_func_proto *fn; enum bpf_arg_type at; + bool full_write; s64 size; if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) return S64_MIN; at = fn->arg_type[arg]; + /* + * Generic outputs may leave bytes untouched. Keep prior initialization + * live when the caller cannot read uninitialized bytes. Constructors of + * special objects, such as dynptrs, still define their storage. + */ + full_write = (at & MEM_UNINIT) && + (!arg_type_is_raw_mem(at) || env->allow_uninit_stack); switch (base_type(at)) { case ARG_PTR_TO_MAP_KEY: @@ -13804,7 +13817,7 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * Size arg is const on each path but differs across merged * paths. MAX_BPF_STACK is a safe upper bound for reads. */ - if (at & MEM_UNINIT) + if (full_write) return 0; return MAX_BPF_STACK; } @@ -13824,10 +13837,10 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn } out: /* - * MEM_UNINIT args are write-only: the helper initializes the - * buffer without reading it. + * Other accesses keep the previous state live, including untouched bytes + * of an unprivileged generic output. */ - if (at & MEM_UNINIT) + if (full_write) return -size; return size; } @@ -13907,7 +13920,8 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn * /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) return -size; - if (is_kfunc_arg_uninit(btf, &args[i])) + if (is_kfunc_arg_uninit(btf, &args[i]) && + (is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack)) return -size; return size; } diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 29260951aa87..195f78db9bda 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -995,7 +995,7 @@ static const struct bpf_func_proto bpf_d_path_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &bpf_d_path_btf_ids[0], - .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .allowed = bpf_d_path_allowed, }; @@ -1052,7 +1052,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = { .func = bpf_snprintf_btf, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, + .arg1_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE, .arg2_type = ARG_MEM_SIZE, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg4_type = ARG_MEM_SIZE, @@ -1565,7 +1565,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = { .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, - .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, + .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE, .arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, };