diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 89dea6b0b024..ebe37c1fe9d2 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -581,20 +581,29 @@ against the arena. Larger accesses must verify the range explicitly. A kfunc may return a scalar, a pointer, or a small struct or union by value. A scalar or pointer of up to 8 bytes is returned in R0, as usual. -A struct or union returned by value must be composed only of scalars -(recursively), where a scalar is an integer or an enum; arrays of scalars are -allowed as members. Its bytes are handed back to the program as the raw -contents of R0 (and R2), so a pointer field would be laundered into a scalar -and escape the verifier's pointer provenance and reference tracking. A struct -or union with a pointer member is therefore rejected at load time, and so is -one with a floating-point member, which the ABI may not return in R0:R2 at +A struct or union returned by value must be composed only of scalars and arena +pointers, where a scalar is an integer or an enum and an arena pointer is one +carrying the ``btf_type_tag("arena")`` attribute. Those may be nested in +structs and unions and in arrays of any number of dimensions, in any +combination, as long as what the nesting bottoms out in is a scalar or an arena +pointer. Its bytes are handed back to the program as the raw contents of R0 +(and R2), so a member of any other pointer type would be laundered into a +scalar and escape the verifier's pointer provenance and reference tracking. A +struct or union with such a member is therefore rejected at load time, and so +is one with a floating-point member, which the ABI may not return in R0:R2 at all. +An arena pointer member is handed back as a scalar too, but nothing is lost by +that. The program must ``cast_kern()`` the value before it can be used, and the +verifier allows that cast on any scalar, so a laundered arena address gives the +program no reach it did not already have. The result is confined to the +program's arena in either case. + A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a -scalar-only struct or union, or an ``__int128``. Such a value is returned -in the register pair R0:R2, matching the convention LLVM uses for the BPF -target: the first 8 bytes in R0 and the second 8 bytes in R2. A struct or -union of 8 bytes or less is returned in R0 alone. +struct or union of scalars and arena pointers, or an ``__int128``. Such a +value is returned in the register pair R0:R2, matching the convention LLVM +uses for the BPF target: the first 8 bytes in R0 and the second 8 bytes in R2. +A struct or union of 8 bytes or less is returned in R0 alone. :: @@ -620,10 +629,10 @@ used when the program is JITed, since the interpreter propagates only R0 out of a subprogram: without a JIT the return value stays in R0 alone, and a caller reading R2 is rejected for reading an uninitialized register. A global subprogram is verified in isolation, so its by-value struct or union return is -restricted to scalars just like a kfunc's; a static subprogram is verified -inline and has no such restriction. The main program is not covered: its return -value is the program's exit code, read out of R0 alone, so a declared upper -half is never looked at. +restricted to scalars and arena pointers just like a kfunc's; a static +subprogram is verified inline and has no such restriction. The main program is +not covered: its return value is the program's exit code, read out of R0 +alone, so a declared upper half is never looked at. A global subprogram must leave a scalar in *every* register of the pair, so both halves of the returned value have to be assigned. Leaving the upper half diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 004b06785521..ae9f606539f4 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,8 +1488,15 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec); + +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f117..ddd0f4f32d24 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -235,6 +235,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 91b8ce77f699..9c2cab08bb79 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3523,6 +3523,22 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena")) + return true; + } + + return false; +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -7927,51 +7943,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; - - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -7979,7 +7963,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ @@ -7989,13 +7973,23 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_struct(t) && t->size <= 16) { /* * A global function's caller models the return as an opaque - * scalar pair, so it may only return scalars by value. A local - * function is verified inline, so a pointer field stays tracked - * and needs no such restriction. + * scalar pair, so a pointer member would be laundered into a + * scalar and escape provenance and reference tracking. Only + * scalars and arena pointers are allowed: an arena pointer has + * no provenance to lose, since a program may already derive one + * from any scalar with addr_space_cast(), which confines the + * result to the arena. The main program is the exception: it + * returns to the kernel, which has no arena to cast the address + * back into. A local function is verified inline, so its caller + * receives the real register state and any member is fine. */ bool local_func = subprog && !is_global; + u32 member_kinds = BTF_MEMBER_SCALAR; - if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + if (subprog) + member_kinds |= BTF_MEMBER_ARENA_PTR; + + if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) return 0; } @@ -8089,10 +8083,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { + /* Only a subprogram may return arena pointers. */ + const char *supported = subprog ? + "void, scalar, arena pointer, or a struct/union of " + "scalars and arena pointers" : + "void, scalar, or a scalar-only struct/union"; + bpf_log(log, "Global function %s() has unsupported return type. " - "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", - tname); + "Only %s up to 16 bytes is supported.\n", + tname, supported); } return err; } @@ -8106,9 +8106,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e036ae20bf6b..a725e3917976 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9995,9 +9995,14 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, */ if (!returns_void) { nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; - for (i = 0; i < nregs; i++) - mark_reg_unknown(env, caller->regs, ret_regs[i]); + mark_reg_unknown(env, caller->regs, ret_regs[0]); bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL, + BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, caller->regs, ret_regs[i]); + bpf_diag_mod_end(env); + } } if (env->subprog_info[subprog].might_throw) { @@ -10403,10 +10408,14 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) * return to the caller whatever the callee had in the * return register(s) */ - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - for (i = 0; i < nregs; i++) - caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; - bpf_diag_mod_end(env); + for (i = 0; i < nregs; i++) { + u32 regno = ret_regs[i]; + + bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], + BPF_DIAG_MOD_WRITE); + caller->regs[regno] = callee->regs[regno]; + bpf_diag_mod_end(env); + } } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -11342,15 +11351,33 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* * Mark the register(s) holding a @size byte kfunc return value as unknown - * scalars. Both halves of a register pair are treated the same way. + * scalars and return how many registers that took. Both halves of a register + * pair are treated the same way. + * + * R0 is written in the diagnostic modification scope the caller opened before + * the caller-saved scrub, since R0 gets no scrub record of its own and that + * scope is what carries its pre-call value into the history. A scope tracks a + * single register and does not nest, so close it before recording the rest of + * the return value. A value that fits in R0 leaves the caller's scope open for + * the caller to close, so a later write to R0 still lands in it. */ -static void mark_kfunc_ret_regs(struct bpf_verifier_env *env, - struct bpf_reg_state *regs, u32 size) +static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) { u32 i, nregs = ret_regs_cnt(size); - for (i = 0; i < nregs; i++) + mark_reg_unknown(env, regs, ret_regs[0]); + if (nregs == 1) + return nregs; + + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, ®s[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE); mark_reg_unknown(env, regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + + return nregs; } static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) @@ -11619,13 +11646,39 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +#define BTF_MEMBER_MAX_DEPTH 4 +#define BTF_MEMBER_PATH_LEN 64 + +struct btf_member_path { + const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; + bool in_array[BTF_MEMBER_MAX_DEPTH]; + const struct btf_type *bad_type; + int depth; + bool too_deep; +}; + +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, 4 levels of nesting allowed. An array member counts as its + * element type. + */ +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds, int rec, + struct btf_member_path *path) { const struct btf_type *member_type; const struct btf_member *member; + bool in_array; u32 i; if (!btf_type_is_struct(t)) @@ -11634,29 +11687,83 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, for_each_member(i, t, member) { const struct btf_array *array; + in_array = false; member_type = btf_type_skip_modifiers(btf, member->type, NULL); + /* + * Every element of an array is laid out in the value being + * returned, so an array counts as its element type however many + * dimensions deep that is. + */ + while (btf_type_is_array(member_type)) { + array = btf_array(member_type); + if (!array->nelems) { + member_type = NULL; + goto bad_member; + } + member_type = btf_type_skip_modifiers(btf, array->type, NULL); + in_array = true; + } if (btf_type_is_struct(member_type)) { - if (rec >= 3) { + if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { verbose(env, "max struct nesting depth exceeded\n"); + if (path) + path->too_deep = true; return false; } - if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) - return false; + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, + rec + 1, path)) + goto bad_path; continue; } - if (btf_type_is_array(member_type)) { - array = btf_array(member_type); - if (!array->nelems) - return false; - member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; - continue; - } - if (!btf_type_is_scalar(member_type)) - return false; + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) + goto bad_member; } return true; + +bad_member: + if (path) { + path->depth = rec + 1; + path->bad_type = member_type; + } +bad_path: + if (path && path->depth) { + path->member[rec] = member; + path->in_array[rec] = in_array; + } + return false; +} + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, u32 member_kinds) +{ + return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); +} + +static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, + char *buf, size_t buf_sz) +{ + size_t len = 0; + int i; + + buf[0] = '\0'; + for (i = 0; i < path->depth; i++) { + const char *name = btf_name_by_offset(btf, path->member[i]->name_off); + + /* An anonymous struct or union has no name to spell. */ + if (!name || !name[0]) + continue; + len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name); + if (path->in_array[i]) + len += scnprintf(buf + len, buf_sz - len, "[]"); + } } enum kfunc_ptr_arg_type { @@ -12039,7 +12146,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12055,7 +12162,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13111,7 +13218,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; } - if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13653,7 +13760,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -13767,7 +13874,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id; + u32 i, nargs, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -14021,25 +14128,63 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_kfunc_ret_regs(env, regs, t->size); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); } else if (btf_type_is_struct(t)) { + struct btf_member_path path = {}; + const char *member_note = ""; + /* * The returned struct comes back as raw register bits modeled - * as an unknown scalar, so it must contain only scalars: - * otherwise a pointer field would be laundered into a scalar - * and escape provenance and reference tracking. + * as an unknown scalar, so a pointer member would be laundered + * into a scalar and escape provenance and reference tracking. + * Only scalars and arena pointers are allowed; see + * btf_validate_return_type() in btf.c for why an arena pointer + * is safe here. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_struct_member_walk(env, desc_btf, t, + BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { verbose(env, - "kernel function %s returns %s %s that is not composed of scalars\n", + "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off)); + if (path.too_deep) { + member_note = bpf_diag_fmt( + env, " It nests structs more than %d levels deep.", + BTF_MEMBER_MAX_DEPTH); + } else if (path.depth) { + char bad_name[BTF_MEMBER_PATH_LEN]; + + btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); + if (!path.bad_type) { + verbose(env, "member '%s' is a zero-length array\n", + bad_name); + member_note = bpf_diag_fmt( + env, " Its member '%s' is a zero-length array.", + bad_name); + } else { + verbose(env, "member '%s' has type %s\n", bad_name, + btf_type_str(path.bad_type)); + member_note = bpf_diag_fmt( + env, + " Its member '%s' is %s, not a scalar or an arena pointer.", + bad_name, btf_type_str(path.bad_type)); + } + } + bpf_diag_program_structure( + env, insn_idx, "unsupported kernel function return type", + "Call a kernel function that returns only scalars or arena pointers by value.", + "%s() returns %s %s by value.%s " + "Only kfuncs returning scalar values or arena pointers, or " + "structures composed of scalar values or arena pointers are " + "supported.", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off), member_note); return -EINVAL; } - mark_kfunc_ret_regs(env, regs, t->size); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -14170,7 +14315,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * Record R0 before process_iter_next_call() snapshots the alternate * iterator path's diagnostic position. */ - bpf_diag_mod_end(env); + if (ret_nregs == 1) + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c index e0b94ed10f94..07d9d6e1d6b8 100644 --- a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c @@ -1,11 +1,53 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include +#include #include "aggregate_ret_func.skel.h" #include "aggregate_ret_kfunc.skel.h" +#include "aggregate_ret_kfunc_arena.skel.h" + +static bool testmod_has_arena_tagged_member(void) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_type *t; + bool tagged = false; + __s32 id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return false; + + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) + goto out; + + /* prog_test_ret_arena::a is 'void __arena_tag *': PTR -> TYPE_TAG -> void */ + id = btf__find_by_name_kind(module_btf, "prog_test_ret_arena", BTF_KIND_STRUCT); + if (id <= 0) + goto out; + + t = btf__type_by_id(module_btf, btf_members(btf__type_by_id(module_btf, id))[0].type); + if (!t || !btf_is_ptr(t)) + goto out; + + t = btf__type_by_id(module_btf, t->type); + tagged = t && btf_is_type_tag(t) && + !strcmp(btf__name_by_offset(module_btf, t->name_off), "arena"); + +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + + return tagged; +} void test_aggregate_ret(void) { RUN_TESTS(aggregate_ret_func); RUN_TESTS(aggregate_ret_kfunc); + + if (testmod_has_arena_tagged_member()) + RUN_TESTS(aggregate_ret_kfunc_arena); + else + test__skip(); } diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c index 6f66fc822ced..f0077d951a25 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include #include +#include #include "bpf_misc.h" typedef unsigned __int128 u128; @@ -234,4 +235,149 @@ __naked int aggregate_ret_global_union_ptr_fail(void) #endif +/* + * gcc returns a by-value struct through a hidden pointer, and emits the + * 'r0 = r1' returning it after the __naked body's exit, leaving the + * subprogram falling through. Build these with clang only. + */ +#if defined(__clang__) + +struct arena_pair { + void __arena *lo; + void __arena *hi; +}; + +struct arena_and_scalar { + void __arena *p; + __u64 x; +}; + +struct arena_array { + void __arena *p[2]; +}; + +struct arena_single { + void __arena *p; +}; + +union arena_upair { + void __arena *p; + __u64 halves[2]; +}; + +__naked struct arena_pair global_ret_arena_pair(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_pair(void) +{ + asm volatile ( + "call %[global_ret_arena_pair];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_pair) + : __clobber_all); +} + +__naked struct arena_and_scalar global_ret_arena_and_scalar(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_and_scalar(void) +{ + asm volatile ( + "call %[global_ret_arena_and_scalar];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_and_scalar) + : __clobber_all); +} + +__naked struct arena_array global_ret_arena_array(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_array(void) +{ + asm volatile ( + "call %[global_ret_arena_array];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_array) + : __clobber_all); +} + +__naked struct arena_single global_ret_arena_single(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__success __retval(0) +__naked int aggregate_ret_global_arena_single(void) +{ + asm volatile ( + "call %[global_ret_arena_single];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_single) + : __clobber_all); +} + +__naked union arena_upair global_ret_arena_union(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_union(void) +{ + asm volatile ( + "call %[global_ret_arena_union];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_union) + : __clobber_all); +} + +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c index d6b422ae9784..0361a7b42c9b 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -19,6 +19,11 @@ void __kfunc_btf_root(void) "r"(&bpf_kfunc_call_test_ret_fastcall), "r"(&bpf_kfunc_call_test_ret_ptr), "r"(&bpf_kfunc_call_test_ret_ii), + "r"(&bpf_kfunc_call_test_ret_nested), + "r"(&bpf_kfunc_call_test_ret_ptr_arr), + "r"(&bpf_kfunc_call_test_ret_deep), + "r"(&bpf_kfunc_call_test_ret_arr_struct), + "r"(&bpf_kfunc_call_test_ret_arr2d), "r"(&bpf_kfunc_call_test_ret_big)); } @@ -71,7 +76,8 @@ __naked int aggregate_ret_kfunc_fastcall_fail(void) SEC("tc") __arch_x86_64 __arch_arm64 -__failure __msg("is not composed of scalars") +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p' has type PTR") __naked int aggregate_ret_kfunc_ptr_fail(void) { asm volatile ( @@ -84,6 +90,82 @@ __naked int aggregate_ret_kfunc_ptr_fail(void) : __clobber_all); } +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in.p' has type PTR") +__naked int aggregate_ret_kfunc_nested_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_nested];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_nested) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p[]' has type PTR") +__naked int aggregate_ret_kfunc_ptr_arr_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_ptr_arr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr_arr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in1[].in2.p' has type PTR") +__naked int aggregate_ret_kfunc_arr_struct_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr_struct];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr_struct) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'a[].p' has type PTR") +__naked int aggregate_ret_kfunc_arr2d_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr2d];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr2d) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("max struct nesting depth exceeded") +__naked int aggregate_ret_kfunc_too_deep_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_deep];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_deep) + : __clobber_all); +} + SEC("tc") __arch_x86_64 __arch_arm64 __failure __msg("R2 !read_ok") diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c new file mode 100644 index 000000000000..a0f263694a63 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena(void *ctx) +{ + u32 volatile __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *a, *b; + struct prog_test_ret_arena r; + + if (!page) + return 1; + + /* Both halves come back in R0:R2, pointing at page and page + 4. */ + r = bpf_kfunc_call_test_ret_arena((u64)page); + if (!r.a || !r.b) + return 2; + + a = (u32 __arena *)r.a; + b = (u32 __arena *)r.b; + *a = 1; + *b = 2; + if (*a != 1) + return 3; + if (*b != 2) + return 4; + + page[0] = 7; + if (*a != 7) + return 5; + page[1] = 9; + if (*b != 9) + return 6; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena_mixed(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + struct prog_test_ret_arena_mixed r; + u32 volatile __arena *p; + + if (!page) + return 1; + + /* An arena pointer in R0 beside a scalar in R2. */ + r = bpf_kfunc_call_test_ret_arena_mixed((u64)page); + if (!r.p) + return 2; + if (r.tag != 0xbeef) + return 3; + + p = (u32 __arena *)r.p; + *p = 3; + if (*p != 3) + return 4; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_untagged_fail(void *ctx) +{ + struct prog_test_ret_arena_untagged r; + + r = bpf_kfunc_call_test_ret_arena_untagged(0); + + return r.a == r.b; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_union_fail(void *ctx) +{ + union prog_test_ret_arena_union r; + + r = bpf_kfunc_call_test_ret_arena_union(0); + + return r.a == r.b; +} + +#else + +SEC("socket") +__description("aggregate_ret_kfunc_arena: needs LLVM 23, dummy test") +__skip("needs LLVM 23") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 9708efb93683..22503cf62e9f 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") +__failure __msg("Only void, scalar, arena pointer, or a struct/union of scalars and arena pointers up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 815f342eb4b0..332322c9b54c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -734,4 +734,52 @@ int check_arena_arg_ret(void *ctx) return 0; } +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct arena_word_pair { + u32 __arena *first; + u32 __arena *second; +}; + +__weak struct arena_word_pair arena_word_pair(u32 __arena *page) +{ + struct arena_word_pair pair; + + pair.first = page; + pair.second = page + 1; + + return pair; +} + +SEC("syscall") +__load_if_JITed() +__success __retval(0) +int check_arena_struct_ret(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *first, *second; + struct arena_word_pair pair; + + if (!page) + return 1; + + pair = arena_word_pair(page); + if (!pair.first || !pair.second) + return 2; + + /* Both halves of the pair must still be usable as arena pointers. */ + first = pair.first; + second = pair.second; + *first = 1; + *second = 2; + if (*first != 1) + return 3; + if (*second != 2) + return 4; + + return 0; +} + +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 850cf4f830c4..2380b6cbdead 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -981,6 +981,69 @@ __bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) return r; } +__bpf_kfunc struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(u64 addr) +{ + struct prog_test_ret_arena r = { .a = (void *)addr, .b = (void *)(addr + 4) }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(u64 addr) +{ + struct prog_test_ret_arena_mixed r = { .p = (void *)addr, .tag = 0xbeef }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(u64 addr) +{ + struct prog_test_ret_arena_untagged r = { .a = (void *)addr, .b = NULL }; + + return r; +} + +__bpf_kfunc union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(u64 addr) +{ + union prog_test_ret_arena_union r = { .a = (void *)addr }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(u64 tag) +{ + struct prog_test_ret_nested r = { .in = { .p = NULL }, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) +{ + struct prog_test_ret_ptr_arr r = { .p = { NULL, NULL } }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) +{ + struct prog_test_ret_arr_struct r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) +{ + struct prog_test_ret_arr2d r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(u64 v) +{ + struct prog_test_ret_deep r = { .l1 = { .l2 = { .l3 = { .l4 = { .v = v } } } } }; + + return r; +} + __bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) { struct prog_test_ret_ii r = { .a = a, .b = b }; @@ -1539,6 +1602,15 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_mixed) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_untagged) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_union) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_nested) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr_arr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr2d) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) #endif BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 65e693ada736..b213ef14848b 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -26,6 +26,12 @@ struct prog_test_ref_kfunc { }; #endif +#if __has_attribute(btf_type_tag) +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#else +#define __arena_tag +#endif + struct bpf_iter_testmod_seq; struct prog_test_pass1 { @@ -70,6 +76,63 @@ struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ __u64 tag; }; +struct prog_test_ret_arena { /* 16 bytes: two arena pointers */ + void __arena_tag *a; + void __arena_tag *b; +}; + +struct prog_test_ret_arena_mixed { /* 16 bytes: an arena pointer and a scalar */ + void __arena_tag *p; + __u64 tag; +}; + +struct prog_test_ret_arena_untagged { /* 16 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + +union prog_test_ret_arena_union { /* 8 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + +struct prog_test_ret_nested { /* 16 bytes: the pointer hides one level down */ + struct { + void *p; + } in; + __u64 tag; +}; + +struct prog_test_ret_ptr_arr { /* 16 bytes: an array of pointers */ + void *p[2]; +}; + +struct prog_test_ret_arr_struct { /* 16 bytes: the pointer is under an array of structs */ + struct { + struct { + void *p; + } in2; + } in1[2]; +}; + +struct prog_test_ret_arr2d { /* 16 bytes: a two dimensional array of structs */ + struct { + void *p; + } a[1][2]; +}; + +struct prog_test_ret_deep { /* 8 bytes, but nested past the 4-level walk limit */ + struct { + struct { + struct { + struct { + __u64 v; + } l4; + } l3; + } l2; + } l1; +}; + struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ __u64 a; __u64 b; @@ -159,6 +222,15 @@ struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; +struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; +struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) __ksym; +struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) __ksym; +struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(__u64 addr) __ksym; +struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(__u64 addr) __ksym; +struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(__u64 addr) __ksym; +union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(__u64 addr) __ksym; +struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(__u64 v) __ksym; struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h,