github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: Preserve stack initialization for generic output buffers

Browse Source 
Partial-output helpers such as bpf_snprintf() do not read incoming buffer
contents, but may leave some bytes untouched. Let them accept uninitialized
storage without promising full initialization to callers that cannot read
uninitialized stack memory.

Make this the default for generic MEM_UNINIT buffers, including __uninit
kfunc arguments. Allow invalid stack bytes through the output check but
leave them invalid when uninitialized stack reads are not allowed. Scrub
initialized bytes and scalar spills as possible writes, retaining the
existing restrictions on spilled pointers and special stack objects.

Retain the output annotation for variable-sized arguments and track their
raw-mode eligibility separately. Callers allowed uninitialized stack reads
can continue treating the potentially written range as initialized. For
constant ranges, defer that initialization until all inputs are checked.

Keep prior stack contents live for generic outputs when the caller cannot
read uninitialized stack memory. Such calls do not define the entire range,
so liveness must preserve initialization facts that remain relevant after
the call. Dynptr and iterator constructors still define their storage.

Annotate the snprintf, sysctl name, d_path, snprintf_btf and branch-record
destinations with MEM_UNINIT, and document that generic __uninit kfuncs may
leave bytes untouched. This also changes readback from existing full-writing
helpers: without permission to read uninitialized stack memory, programs
must initialize those bytes themselves before reading them after a call.

Suggested-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://patch.msgid.link/20260921023843.411943-9-memxor@gmail.com
This commit is contained in:
Kumar Kartikeya Dwivedi 2026-09-21 04:38:32 +02:00 committed by Alexei Starovoitov
parent 89835699ef
commit 5da4a9f26f
No known key found for this signature in database
7 changed files with 54 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Documentation/bpf/kfuncs.rst
View File

@ -164,21 +164,22 @@ suffix should be used.
2.3.3 __uninit Annotation 2.3.3 __uninit Annotation
------------------------- -------------------------
Use ``__uninit`` on a pointer parameter for an output that the kfunc Use ``__uninit`` on a pointer parameter for an output buffer whose incoming
initializes without reading its incoming contents. contents the kfunc does not read.
For generic memory buffers, the kfunc must initialize every byte in the For generic memory buffers, the kfunc may leave bytes untouched, including
declared range on every return path, including error returns and struct on error returns. The writable range is determined by the pointed-to type
padding. The range is determined by the pointed-to type or the associated or the associated ``__sz`` or ``__szk`` size argument.
``__sz`` or ``__szk`` size argument.
The annotation does not change the accepted pointer types. A stack-backed The annotation does not change the accepted pointer types. A stack-backed
struct passed as a generic memory buffer must still be scalar-only. struct passed as a generic memory buffer must still be scalar-only.
A stack buffer with a verifier-known constant offset and size may be Generic stack buffers may be uninitialized before the call. The call does
uninitialized before the call and is considered initialized afterwards. not make previously uninitialized bytes readable unless the program is
For variable offsets or sizes, the usual stack-initialization and allowed to read uninitialized stack memory (normally requiring
variable-offset restrictions still apply. ``CAP_PERFMON``). Other callers must initialize those bytes themselves
before reading them. Stack bounds and variable-offset restrictions still
apply.
For dynptr parameters, ``__uninit`` indicates that the kfunc constructs a For dynptr parameters, ``__uninit`` indicates that the kfunc constructs a
dynptr in the supplied storage. For example:: dynptr in the supplied storage. For example::

5
include/linux/bpf.h
View File

@ -781,7 +781,10 @@ enum bpf_type_flag {
*/ */
PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS),
/* MEM can be uninitialized. */ /*
* MEM can be uninitialized. Generic memory outputs need not be fully
* initialized by the callee.
*/
MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS),
/* DYNPTR points to memory local to the bpf program. */ /* DYNPTR points to memory local to the bpf program. */

8
include/linux/bpf_verifier.h
View File

@ -1547,12 +1547,14 @@ struct ref_obj_desc {
}; };
/* /*
* Memory arguments a call fills in, indexed by argument slot. The verifier allows the * Generic MEM_UNINIT arguments, indexed by ABI slot. var_size_mask excludes
* stack to be uninitialized if the range is a known constant. Stack slots are marked as * variable-sized buffers from raw mode without losing the output annotation.
* STACK_MISC by check_mem_access() after all arguments have been checked. * size records constant ranges to mark initialized after checking all arguments,
* only when the caller is allowed to read uninitialized stack memory.
*/ */
struct arg_raw_mem_desc { struct arg_raw_mem_desc {
u16 mask; u16 mask;
u16 var_size_mask;
int size[MAX_BPF_FUNC_ARGS]; int size[MAX_BPF_FUNC_ARGS];
}; };

2
kernel/bpf/cgroup.c
View File

@ -2491,7 +2491,7 @@ static const struct bpf_func_proto bpf_sysctl_get_name_proto = {
.gpl_only = false, .gpl_only = false,
.ret_type = RET_INTEGER, .ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX, .arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE,
.arg3_type = ARG_MEM_SIZE, .arg3_type = ARG_MEM_SIZE,
.arg4_type = ARG_ANYTHING, .arg4_type = ARG_ANYTHING,
}; };

2
kernel/bpf/helpers.c
View File

@ -1130,7 +1130,7 @@ const struct bpf_func_proto bpf_snprintf_proto = {
.func = bpf_snprintf, .func = bpf_snprintf,
.gpl_only = true, .gpl_only = true,
.ret_type = RET_INTEGER, .ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg1_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE,
.arg2_type = ARG_MEM_SIZE_OR_ZERO, .arg2_type = ARG_MEM_SIZE_OR_ZERO,
.arg3_type = ARG_PTR_TO_CONST_STR, .arg3_type = ARG_PTR_TO_CONST_STR,
.arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,

44
kernel/bpf/verifier.c
View File

@ -6993,10 +6993,11 @@ static int check_stack_range_initialized(
* but BTF based global subprog validation isn't accurate enough. * but BTF based global subprog validation isn't accurate enough.
*/ */
bool allow_poison = access_size < 0 || clobber; bool allow_poison = access_size < 0 || clobber;
/* The call will initialize the memory; uninitialized stack allowed */
u32 arg_slot = arg_slot_from_argno(argno); u32 arg_slot = arg_slot_from_argno(argno);
bool raw_mode = meta && arg_slot < MAX_BPF_FUNC_ARGS && bool uninit = clobber && meta && arg_slot < MAX_BPF_FUNC_ARGS &&
(meta->arg_raw_mem.mask & BIT(arg_slot)); (meta->arg_raw_mem.mask & BIT(arg_slot));
bool raw_mode = uninit && env->allow_uninit_stack &&
!(meta->arg_raw_mem.var_size_mask & BIT(arg_slot));
access_size = abs(access_size); access_size = abs(access_size);
@ -7035,6 +7036,7 @@ static int check_stack_range_initialized(
max_off = reg_smax(reg) + off; max_off = reg_smax(reg) + off;
} }
/* Unprivileged outputs retain each byte's initialization state. */
if (raw_mode) { if (raw_mode) {
meta->arg_raw_mem.size[arg_slot] = access_size; meta->arg_raw_mem.size[arg_slot] = access_size;
return 0; return 0;
@ -7054,8 +7056,8 @@ static int check_stack_range_initialized(
if (*stype == STACK_MISC) if (*stype == STACK_MISC)
goto mark; goto mark;
if ((*stype == STACK_ZERO) || if ((*stype == STACK_ZERO) ||
(*stype == STACK_INVALID && env->allow_uninit_stack)) { (*stype == STACK_INVALID && (uninit || env->allow_uninit_stack))) {
if (clobber) { if (clobber && (*stype != STACK_INVALID || env->allow_uninit_stack)) {
/* helper can write anything into the stack */ /* helper can write anything into the stack */
*stype = STACK_MISC; *stype = STACK_MISC;
} }
@ -7074,8 +7076,11 @@ static int check_stack_range_initialized(
} }
if (*stype == STACK_POISON) { if (*stype == STACK_POISON) {
if (allow_poison) if (allow_poison) {
if (uninit && env->allow_uninit_stack)
*stype = STACK_MISC;
goto mark; goto mark;
}
verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n", verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n",
reg_arg_name(env, argno), min_off, i - min_off, access_size); reg_arg_name(env, argno), min_off, i - min_off, access_size);
} else if (tnum_is_const(reg->var_off)) { } else if (tnum_is_const(reg->var_off)) {
@ -7224,12 +7229,12 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
meta->msize_max_value = reg_umax(size_reg); meta->msize_max_value = reg_umax(size_reg);
/* /*
* A variable size does not guarantee that the call initializes the whole * Check variable ranges byte by byte instead of using raw mode. Keep the
* checked range. Disable raw mode for this output and apply the ordinary * MEM_UNINIT annotation so invalid bytes are accepted without marking them
* stack initialization checks, including their privilege exceptions. * initialized when the caller cannot read uninitialized stack memory.
*/ */
if (!tnum_is_const(size_reg->var_off)) if (!tnum_is_const(size_reg->var_off))
meta->arg_raw_mem.mask &= ~BIT(arg_slot_from_argno(mem_argno)); meta->arg_raw_mem.var_size_mask |= BIT(arg_slot_from_argno(mem_argno));
if (reg_smin(size_reg) < 0) { if (reg_smin(size_reg) < 0) {
verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n",
@ -13730,12 +13735,20 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn
struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
const struct bpf_func_proto *fn; const struct bpf_func_proto *fn;
enum bpf_arg_type at; enum bpf_arg_type at;
bool full_write;
s64 size; s64 size;
if (bpf_get_helper_proto(env, insn->imm, &fn) < 0) if (bpf_get_helper_proto(env, insn->imm, &fn) < 0)
return S64_MIN; return S64_MIN;
at = fn->arg_type[arg]; at = fn->arg_type[arg];
/*
* Generic outputs may leave bytes untouched. Keep prior initialization
* live when the caller cannot read uninitialized bytes. Constructors of
* special objects, such as dynptrs, still define their storage.
*/
full_write = (at & MEM_UNINIT) &&
(!arg_type_is_raw_mem(at) || env->allow_uninit_stack);
switch (base_type(at)) { switch (base_type(at)) {
case ARG_PTR_TO_MAP_KEY: case ARG_PTR_TO_MAP_KEY:
@ -13804,7 +13817,7 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn
* Size arg is const on each path but differs across merged * Size arg is const on each path but differs across merged
* paths. MAX_BPF_STACK is a safe upper bound for reads. * paths. MAX_BPF_STACK is a safe upper bound for reads.
*/ */
if (at & MEM_UNINIT) if (full_write)
return 0; return 0;
return MAX_BPF_STACK; return MAX_BPF_STACK;
} }
@ -13824,10 +13837,10 @@ s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn
} }
out: out:
/* /*
* MEM_UNINIT args are write-only: the helper initializes the * Other accesses keep the previous state live, including untouched bytes
* buffer without reading it. * of an unprivileged generic output.
*/ */
if (at & MEM_UNINIT) if (full_write)
return -size; return -size;
return size; return size;
} }
@ -13907,7 +13920,8 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *
/* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */ /* KF_ITER_NEW kfuncs initialize the iterator state at arg 0 */
if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW) if (arg == 0 && meta.kfunc_flags & KF_ITER_NEW)
return -size; return -size;
if (is_kfunc_arg_uninit(btf, &args[i])) if (is_kfunc_arg_uninit(btf, &args[i]) &&
(is_kfunc_arg_dynptr(btf, &args[i]) || env->allow_uninit_stack))
return -size; return -size;
return size; return size;
} }

6
kernel/trace/bpf_trace.c
View File

@ -995,7 +995,7 @@ static const struct bpf_func_proto bpf_d_path_proto = {
.ret_type = RET_INTEGER, .ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_BTF_ID, .arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &bpf_d_path_btf_ids[0], .arg1_btf_id = &bpf_d_path_btf_ids[0],
.arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg2_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE,
.arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.allowed = bpf_d_path_allowed, .allowed = bpf_d_path_allowed,
}; };
@ -1052,7 +1052,7 @@ const struct bpf_func_proto bpf_snprintf_btf_proto = {
.func = bpf_snprintf_btf, .func = bpf_snprintf_btf,
.gpl_only = false, .gpl_only = false,
.ret_type = RET_INTEGER, .ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, .arg1_type = ARG_PTR_TO_MEM | MEM_UNINIT | MEM_WRITE,
.arg2_type = ARG_MEM_SIZE, .arg2_type = ARG_MEM_SIZE,
.arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY,
.arg4_type = ARG_MEM_SIZE, .arg4_type = ARG_MEM_SIZE,
@ -1565,7 +1565,7 @@ static const struct bpf_func_proto bpf_read_branch_records_proto = {
.gpl_only = true, .gpl_only = true,
.ret_type = RET_INTEGER, .ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX, .arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_UNINIT | MEM_WRITE,
.arg3_type = ARG_MEM_SIZE_OR_ZERO, .arg3_type = ARG_MEM_SIZE_OR_ZERO,
.arg4_type = ARG_ANYTHING, .arg4_type = ARG_ANYTHING,
}; };