diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 004e2ddeb4f6..11a4e4f87aac 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2359,6 +2359,16 @@ static const u8 x86_arg_reg[] = { BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5, X86_REG_R9, }; +static bool x86_arg_on_stack(u8 pos) +{ + return pos >= x86_arg_abi.nr_arg_regs; +} + +static s32 x86_arg_stack_off(u8 pos) +{ + return (pos - x86_arg_abi.nr_arg_regs) * 8; +} + /* * Move the arguments the x86-64 ABI places somewhere other than the argument * slot the BPF calling convention gave them. @stack_base addresses the @@ -3883,21 +3893,17 @@ static void clean_stack_garbage(const struct btf_func_model *m, } } -/* get the count of the regs that are used to pass arguments */ -static int get_nr_used_regs(const struct btf_func_model *m) +/* the eightbytes taken by the on-stack arguments, alignment holes included */ +static int get_nr_stack_slots(const u8 *pos_of_slot, int nr_slots) { - int i, arg_regs, nr_used_regs = 0; + int slot; - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { - arg_regs = (m->arg_size[i] + 7) / 8; - if (nr_used_regs + arg_regs <= 6) - nr_used_regs += arg_regs; - - if (nr_used_regs >= 6) - break; + for (slot = nr_slots - 1; slot >= 0; slot--) { + if (x86_arg_on_stack(pos_of_slot[slot])) + return x86_arg_stack_off(pos_of_slot[slot]) / 8 + 1; } - return nr_used_regs; + return 0; } /* @@ -3927,20 +3933,21 @@ static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base } static void save_args(const struct btf_func_model *m, u8 **prog, - int stack_size, bool for_call_origin, u32 flags, - u64 arena_base) + const u8 *pos_of_slot, int stack_size, + bool for_call_origin, u32 flags, u64 arena_base) { - int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0; + int arg_regs, first_off = 0, nr_stack_slots = 0; bool use_jmp = bpf_trampoline_use_jmp(flags); int stack_args_off = (use_jmp || (flags & BPF_TRAMP_F_INDIRECT)) ? 16 : 24; - int i, j; + int bargs_off = stack_size; + int i, j, slot, off, doff; /* Store function arguments to stack. * For a function that accepts two pointers the sequence will be: * mov QWORD PTR [rbp-0x10],rdi * mov QWORD PTR [rbp-0x8],rsi */ - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { + for (i = 0, slot = 0; i < m->nr_args; i++) { bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG); bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG; @@ -3962,8 +3969,36 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * * the arg1-5,arg7 will be passed by regs, and arg6 will * by stack. + * + * An argument aligned to 16 bytes, an __int128 or a struct + * holding one, also starts at an even stack slot, which can + * leave a hole before it. bpf_jit_place_args() worked out + * where each eightbyte is. */ - if (nr_regs + arg_regs > 6) { + for (j = 0; j < arg_regs; j++, slot++, bargs_off -= 8) { + u8 pos = pos_of_slot[slot]; + u32 src; + + if (!x86_arg_on_stack(pos)) { + /* + * Only copy the arguments on-stack to + * current 'stack_size' and ignore the regs, + * used to prepare the arguments on-stack for + * origin call. + */ + if (for_call_origin) + continue; + + /* copy the arguments from regs into stack */ + src = x86_arg_reg[pos]; + if (arena_arg) { + emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); + src = BPF_REG_0; + } + emit_stx(prog, BPF_DW, BPF_REG_FP, src, -bargs_off); + continue; + } + /* copy function arguments from origin stack frame * into current stack frame. * @@ -3973,53 +4008,33 @@ static void save_args(const struct btf_func_model *m, u8 **prog, * entered through the fentry call, so rbp + 24, and * a single one when it is entered with a jmp or * called indirectly, so rbp + 16. + * + * The area for the origin call mirrors the incoming + * one, hole and all; only the bpf program takes the + * arguments packed. */ - for (j = 0; j < arg_regs; j++) { - emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, - nr_stack_slots * 8 + stack_args_off); - if (arena_arg) - emit_arena_arg_conv(prog, BPF_REG_0, nullable, - (u32)arena_base); - emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, - -stack_size); + off = x86_arg_stack_off(pos); + doff = for_call_origin ? stack_size - off : bargs_off; + emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP, + stack_args_off + off); + if (arena_arg) + emit_arena_arg_conv(prog, BPF_REG_0, nullable, + (u32)arena_base); + emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -doff); - if (!nr_stack_slots) - first_off = stack_size; - stack_size -= 8; - nr_stack_slots++; - } - } else { - /* Only copy the arguments on-stack to current - * 'stack_size' and ignore the regs, used to - * prepare the arguments on-stack for origin call. - */ - if (for_call_origin) { - nr_regs += arg_regs; - continue; - } - - /* copy the arguments from regs into stack */ - for (j = 0; j < arg_regs; j++) { - u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs; - - if (arena_arg) { - emit_arena_arg_conv(prog, src, nullable, (u32)arena_base); - src = BPF_REG_0; - } - emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size); - stack_size -= 8; - nr_regs++; - } + if (!nr_stack_slots) + first_off = doff; + nr_stack_slots++; } } clean_stack_garbage(m, prog, nr_stack_slots, first_off); } -static void restore_regs(const struct btf_func_model *m, u8 **prog, +static void restore_regs(u8 **prog, const u8 *pos_of_slot, int nr_slots, int stack_size) { - int i, j, arg_regs, nr_regs = 0; + int slot; /* Restore function arguments from stack. * For a function that accepts two pointers the sequence will be: @@ -4028,23 +4043,10 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, * * The logic here is similar to what we do in save_args() */ - for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { - arg_regs = (m->arg_size[i] + 7) / 8; - if (nr_regs + arg_regs <= 6) { - for (j = 0; j < arg_regs; j++) { - emit_ldx(prog, BPF_DW, - nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs, - BPF_REG_FP, - -stack_size); - stack_size -= 8; - nr_regs++; - } - } else { - stack_size -= 8 * arg_regs; - } - - if (nr_regs >= 6) - break; + for (slot = 0; slot < nr_slots; slot++, stack_size -= 8) { + if (!x86_arg_on_stack(pos_of_slot[slot])) + emit_ldx(prog, BPF_DW, x86_arg_reg[pos_of_slot[slot]], + BPF_REG_FP, -stack_size); } } @@ -4233,9 +4235,9 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, return 0; } -/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */ -#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack) \ - __LOAD_TCC_PTR(-round_up(stack, 8) - 8) +/* mov rax, qword ptr [rbp - off] */ +#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(off) \ + __LOAD_TCC_PTR(-(off)) /* Example: * __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev); @@ -4303,8 +4305,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im struct bpf_tramp_nodes *tnodes, void *func_addr) { - int i, ret, nr_regs = m->nr_args, stack_size = 0; + int i, ret, nr_regs, stack_size = 0; + u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS]; int regs_off, func_meta_off, ip_off, run_ctx_off, arg_stack_off, rbx_off; + int tcc_off = 0; struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; @@ -4326,12 +4330,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im arena_base = bpf_tramp_arena_base(m, tnodes, flags); - for (i = 0; i < m->nr_args; i++) - nr_regs += (m->arg_size[i] + 7) / 8 - 1; - - /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 - * are passed through regs, the remains are through stack. + /* + * x86-64 supports up to MAX_BPF_FUNC_ARGS argument slots, six in + * registers and the rest on the stack, where bpf_jit_place_args() + * puts them. */ + nr_regs = bpf_jit_place_args(&x86_arg_abi, m, pos_of_slot); if (nr_regs > MAX_BPF_FUNC_ARGS) return -ENOTSUPP; @@ -4343,11 +4347,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * RBP - 8 [ return value ] BPF_TRAMP_F_CALL_ORIG or * BPF_TRAMP_F_RET_FENTRY_RET flags * - * [ reg_argN ] always - * [ ... ] - * RBP - regs_off [ reg_arg1 ] program's ctx pointer + * [ arg slotN ] always, packed, whether + * [ ... ] in a register or on the + * RBP - regs_off [ arg slot1 ] stack; program's ctx pointer * - * RBP - func_meta_off [ regs count, etc ] always + * RBP - func_meta_off [ slot count, etc ] always * * RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag * @@ -4355,11 +4359,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * * RBP - run_ctx_off [ bpf_tramp_run_ctx ] * - * [ stack_argN ] BPF_TRAMP_F_CALL_ORIG - * [ ... ] - * [ stack_arg2 ] - * RBP - arg_stack_off [ stack_arg1 ] - * RSP [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX + * RBP - tcc_off [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX + * + * [ stack slotN ] BPF_TRAMP_F_CALL_ORIG, + * [ ... ] the incoming stack + * [ stack slot2 ] arguments, alignment + * RBP - arg_stack_off [ stack slot1 ] holes included */ /* room for return value of orig_call or fentry prog */ @@ -4370,7 +4375,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im stack_size += nr_regs * 8; regs_off = stack_size; - /* function matedata, such as regs count */ + /* function matedata, such as the argument slot count */ stack_size += 8; func_meta_off = stack_size; @@ -4390,9 +4395,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7; run_ctx_off = stack_size; + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_off = stack_size; + } + if (nr_regs > 6 && (flags & BPF_TRAMP_F_CALL_ORIG)) { /* the space that used to pass arguments on-stack */ - stack_size += (nr_regs - get_nr_used_regs(m)) * 8; + stack_size += get_nr_stack_slots(pos_of_slot, nr_regs) * 8; /* make sure the stack pointer is 16-byte aligned if we * need pass arguments on stack, which means * [stack_size + 8(rbp) + 8(rip) + 8(origin rip)] @@ -4442,12 +4452,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* sub rsp, stack_size */ emit_sub_rsp(&prog, stack_size); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) - EMIT1(0x50); /* push rax */ + /* mov QWORD PTR [rbp - tcc_off], rax */ + emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -tcc_off); /* mov QWORD PTR [rbp - rbx_off], rbx */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); func_meta = nr_regs; - /* Store number of argument registers of the traced function */ + /* Store number of argument slots of the traced function */ emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta); if (flags & BPF_TRAMP_F_IP_ARG) { @@ -4455,7 +4466,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr); } - save_args(m, &prog, regs_off, false, flags, arena_base); + save_args(m, &prog, pos_of_slot, regs_off, false, flags, arena_base); if (flags & BPF_TRAMP_F_CALL_ORIG) { /* arg1: mov rdi, im */ @@ -4496,14 +4507,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_CALL_ORIG) { - restore_regs(m, &prog, regs_off); - save_args(m, &prog, arg_stack_off, true, flags, 0); + restore_regs(&prog, pos_of_slot, nr_regs, regs_off); + save_args(m, &prog, pos_of_slot, arg_stack_off, true, flags, 0); if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { /* Before calling the original function, load the * tail_call_cnt_ptr from stack to rax. */ - LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); + LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off); } if (flags & BPF_TRAMP_F_ORIG_STACK) { @@ -4550,7 +4561,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_RESTORE_REGS) - restore_regs(m, &prog, regs_off); + restore_regs(&prog, pos_of_slot, nr_regs, regs_off); /* This needs to be done regardless. If there were fmod_ret programs, * the return value is only updated on the stack and still needs to be @@ -4567,7 +4578,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Before running the original function, load the * tail_call_cnt_ptr from stack to rax. */ - LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); + LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off); } /* restore return value of orig_call or fentry prog back into RAX */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 62f8951126ba..ddfa0c6d2755 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -22006,6 +22006,59 @@ static bool attach_uses_trampoline_retval(enum bpf_attach_type type) } } +/* + * BPF code passes argument slot N where the kernel convention passes its Nth + * u64 argument, so describe a BPF function to its trampoline as one u64 per + * slot, and every arch places its arguments right. + */ +static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tname, + struct btf_func_model *m) +{ + u32 i, nslots = 0; + + for (i = 0; i < m->nr_args; i++) + nslots += btf_func_model_arg_slots(m, i); + if (nslots > MAX_BPF_FUNC_ARGS) { + bpf_log(log, "The function %s has %u argument slots. Too many.\n", + tname, nslots); + return -EINVAL; + } + + for (i = 0; i < nslots; i++) { + m->arg_size[i] = sizeof(u64); + m->arg_flags[i] = 0; + } + m->nr_args = nslots; + return 0; +} + +/* + * The verifier lets a static function use stack arguments counted from its + * BTF, even if that BTF turns out unreliable. A trampoline on the function is + * then given only MAX_BPF_FUNC_REG_ARGS u64 arguments and would lose the + * stack arguments, so such a function cannot be attached to. + */ +static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf *btf, + const struct btf_type *func_proto, const char *tname) +{ + struct btf_func_model m; + u32 i, nslots = 0; + + if (btf_distill_func_proto(NULL, btf, func_proto, tname, &m)) { + /* at least a slot for each argument */ + nslots = btf_type_vlen(func_proto); + } else { + for (i = 0; i < m.nr_args; i++) + nslots += btf_func_model_arg_slots(&m, i); + } + if (nslots <= MAX_BPF_FUNC_REG_ARGS) + return 0; + + bpf_log(log, "Cannot attach to static function %s: unreliable BTF and stack arguments\n", + tname); + return -EINVAL; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -22273,12 +22326,23 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) return -EINVAL; - if (tgt_prog && conservative) + if (tgt_prog && conservative) { + if (subprog > 0) { + ret = check_unreliable_func_args(log, btf, t, tname); + if (ret) + return ret; + } t = NULL; + } ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel); if (ret < 0) return ret; + if (tgt_prog) { + ret = flatten_bpf_func_model(log, tname, &tgt_info->fmodel); + if (ret < 0) + return ret; + } if (tgt_info->fmodel.ret_size > 8 && attach_uses_trampoline_retval(prog->expected_attach_type)) { diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c index 15b95d0235b5..de358f335d98 100644 --- a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c +++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c @@ -5,6 +5,10 @@ #include "tracing_struct.skel.h" #include "tracing_struct_many_args.skel.h" #include "tracing_struct_int128.skel.h" +#include "tracing_struct_int128_tgt.skel.h" +#include "tracing_struct_int128_tracer.skel.h" +#include "tracing_stack_arg_tailcall_tgt.skel.h" +#include "tracing_stack_arg_tailcall_tracer.skel.h" static void test_struct_args(void) { @@ -117,9 +121,11 @@ static void test_int128_args(void) { /* * __int128 arguments are passed in a register pair on x86_64 and - * arm64, which the trampoline packs into two context slots. Other - * architectures pass a __int128 differently (e.g. s390x passes larger - * arguments by reference), so only exercise this on x86_64 and arm64. + * arm64, which the trampoline packs into two context slots. Past the + * registers, both start an argument aligned to 16 bytes at an even + * stack slot, which the trampoline packs too. Other architectures + * pass a __int128 differently (e.g. s390x passes larger arguments by + * reference), so only exercise this on x86_64 and arm64. */ #if defined(__x86_64__) || defined(__aarch64__) struct tracing_struct_int128 *skel; @@ -139,6 +145,20 @@ static void test_int128_args(void) ASSERT_EQ(skel->bss->t_c, 3, "t:c"); ASSERT_EQ(skel->bss->t_ret, 6, "t ret"); + ASSERT_EQ(skel->bss->s_g, 7, "s:g"); + ASSERT_EQ(skel->bss->s_h_lo, 8, "s:h.lo"); + ASSERT_EQ(skel->bss->s_h_hi, 9, "s:h.hi"); + ASSERT_EQ(skel->bss->s_i, 10, "s:i"); + ASSERT_EQ(skel->bss->s_ret, 385, "s ret"); + + ASSERT_EQ(skel->bss->b_f_a, 6, "b:f.a"); + ASSERT_EQ(skel->bss->b_f_b, 7, "b:f.b"); + ASSERT_EQ(skel->bss->b_g, 8, "b:g"); + ASSERT_EQ(skel->bss->b_h, 9, "b:h"); + ASSERT_EQ(skel->bss->b_i_lo, 10, "b:i.lo"); + ASSERT_EQ(skel->bss->b_i_hi, 11, "b:i.hi"); + ASSERT_EQ(skel->bss->b_ret, 506, "b ret"); + destroy_skel: tracing_struct_int128__destroy(skel); #else @@ -146,6 +166,191 @@ static void test_int128_args(void) #endif } +static void test_int128_tgt_prog(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) || defined(__s390x__) + struct tracing_struct_int128_tracer *tracer = NULL; + struct tracing_struct_int128_tgt *tgt; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + tgt = tracing_struct_int128_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_struct_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_struct_int128_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + err = bpf_program__set_attach_target(tracer->progs.odd_entry, fd, "callee_odd"); + err = err ?: bpf_program__set_attach_target(tracer->progs.odd_exit, fd, "callee_odd"); + if (tgt->rodata->has_stack_arg) { + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_entry, fd, + "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, + "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.split_entry, fd, + "callee_split"); + err = err ?: bpf_program__set_attach_target(tracer->progs.split_exit, fd, + "callee_split"); + } else { + bpf_program__set_autoload(tracer->progs.stack_entry, false); + bpf_program__set_autoload(tracer->progs.stack_exit, false); + bpf_program__set_autoload(tracer->progs.split_entry, false); + bpf_program__set_autoload(tracer->progs.split_exit, false); + } + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_struct_int128_tracer__load(tracer); + if (!ASSERT_OK(err, "tracer__load")) + goto out; + err = tracing_struct_int128_tracer__attach(tracer); + if (!ASSERT_OK(err, "tracer__attach")) + goto out; + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(tracer->bss->odd_a, 1, "odd:a"); + ASSERT_EQ(tracer->bss->odd_lo, 8, "odd:v.lo"); + ASSERT_EQ(tracer->bss->odd_hi, 9, "odd:v.hi"); + ASSERT_EQ(tracer->bss->odd_b, 2, "odd:b"); + ASSERT_EQ(tgt->bss->ret_odd, 52, "odd ret"); + ASSERT_EQ(tracer->bss->odd_ret, 52, "odd fexit ret"); + + if (tgt->rodata->has_stack_arg) { + ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g"); + ASSERT_EQ(tracer->bss->stack_lo, 8, "stack:h.lo"); + ASSERT_EQ(tracer->bss->stack_hi, 9, "stack:h.hi"); + ASSERT_EQ(tracer->bss->stack_i, 10, "stack:i"); + ASSERT_EQ(tgt->bss->ret_stack, 385, "stack ret"); + ASSERT_EQ(tracer->bss->stack_ret, 385, "stack fexit ret"); + + ASSERT_EQ(tracer->bss->split_e, 5, "split:e"); + ASSERT_EQ(tracer->bss->split_lo, 8, "split:h.lo"); + ASSERT_EQ(tracer->bss->split_hi, 9, "split:h.hi"); + ASSERT_EQ(tracer->bss->split_i, 10, "split:i"); + ASSERT_EQ(tgt->bss->ret_split, 246, "split ret"); + ASSERT_EQ(tracer->bss->split_ret, 246, "split fexit ret"); + } + +out: + tracing_struct_int128_tracer__destroy(tracer); + tracing_struct_int128_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + +/* + * A static function whose BTF is unreliable gets a trampoline that only knows + * of the register arguments, so one taking stack arguments must be refused. + */ +static void test_tgt_prog_unreliable(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_struct_int128_tracer *tracer = NULL; + struct tracing_struct_int128_tgt *tgt; + char log_buf[4096] = {}; + struct bpf_program *prog; + int err, fd; + + tgt = tracing_struct_int128_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_stack_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_struct_int128_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + bpf_object__for_each_program(prog, tracer->obj) + bpf_program__set_autoload(prog, false); + bpf_program__set_autoload(tracer->progs.unreliable_entry, true); + bpf_program__set_log_buf(tracer->progs.unreliable_entry, log_buf, sizeof(log_buf)); + err = bpf_program__set_attach_target(tracer->progs.unreliable_entry, fd, + "callee_unreliable"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_struct_int128_tracer__load(tracer); + ASSERT_ERR(err, "tracer__load"); + ASSERT_HAS_SUBSTR(log_buf, "unreliable BTF and stack arguments", "log"); + +out: + tracing_struct_int128_tracer__destroy(tracer); + tracing_struct_int128_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + +/* + * A trampoline on a function of a tail_call_reachable program keeps the tail + * call counter pointer on its stack, and has to hand the function its stack + * arguments past it. + */ +static void test_stack_arg_tailcall(void) +{ +#if defined(__x86_64__) || defined(__aarch64__) + struct tracing_stack_arg_tailcall_tracer *tracer = NULL; + struct tracing_stack_arg_tailcall_tgt *tgt; + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err, fd; + + tgt = tracing_stack_arg_tailcall_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load")) + return; + if (!tgt->rodata->has_stack_arg) { + test__skip(); + goto out; + } + fd = bpf_program__fd(tgt->progs.run); + + tracer = tracing_stack_arg_tailcall_tracer__open(); + if (!ASSERT_OK_PTR(tracer, "tracer__open")) + goto out; + + err = bpf_program__set_attach_target(tracer->progs.stack_entry, fd, "callee_stack"); + err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, "callee_stack"); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = tracing_stack_arg_tailcall_tracer__load(tracer); + if (!ASSERT_OK(err, "tracer__load")) + goto out; + err = tracing_stack_arg_tailcall_tracer__attach(tracer); + if (!ASSERT_OK(err, "tracer__attach")) + goto out; + + err = bpf_prog_test_run_opts(fd, &topts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval")) + goto out; + + ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g"); + ASSERT_EQ(tracer->bss->stack_h, 8, "stack:h"); + ASSERT_EQ(tgt->bss->ret_stack, 204, "stack ret"); + ASSERT_EQ(tracer->bss->stack_ret, 204, "stack fexit ret"); + +out: + tracing_stack_arg_tailcall_tracer__destroy(tracer); + tracing_stack_arg_tailcall_tgt__destroy(tgt); +#else + test__skip(); +#endif +} + static void test_union_args(void) { struct tracing_struct *skel; @@ -181,6 +386,12 @@ void test_tracing_struct(void) test_struct_many_args(); if (test__start_subtest("int128_args")) test_int128_args(); + if (test__start_subtest("int128_tgt_prog")) + test_int128_tgt_prog(); + if (test__start_subtest("tgt_prog_unreliable")) + test_tgt_prog_unreliable(); + if (test__start_subtest("stack_arg_tailcall")) + test_stack_arg_tailcall(); if (test__start_subtest("union_args")) test_union_args(); } diff --git a/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c index 52d3d5d53946..c87e886423f4 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_arg_kfunc.c @@ -50,6 +50,27 @@ int aggregate_arg_kfunc_int128(struct __sk_buff *skb) return 0; } +/* + * arm64 rounds the register number up to an even one for an argument + * aligned to 16 bytes, so it wants this __int128 in x2 and x3, leaving x1 + * empty, and the last argument in x4. The x86-64 ABI has no such rule. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_odd(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + + if (bpf_kfunc_call_test_i128_arg_odd(1, v, 2) != 1 + 2 * b + 3 * a + 4 * 2) + return 1; + + return 0; +} + #endif /* __SIZEOF_INT128__ */ #if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) @@ -131,6 +152,56 @@ int aggregate_arg_kfunc_split8(struct __sk_buff *skb) #ifdef __SIZEOF_INT128__ +/* + * The hole of aggregate_arg_kfunc_int128_odd, with arguments after it that + * arm64 shifts through x5 to x7, where its JIT keeps the BPF stack slots, + * and onto its stack. The second __int128 then leaves the second arm64 + * stack slot empty. x86-64 packs the eightbytes like the BPF convention, + * so nothing moves there. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_odd_many(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v = ((u128)a << 64) | b; + u128 w = ((u128)~a << 64) | ~b; + + if (bpf_kfunc_call_test_i128_arg_odd_many(1, v, 2, 3, 4, 5, 6, w) != + 1 + 2 * b + 3 * a + 4 * 2 + 5 * 3 + 6 * 4 + 7 * 5 + 8 * 6 + + 9 * ~b + 10 * ~a) + return 1; + + return 0; +} + +/* + * x86-64 moves s to the stack and gives R9 back to the seventh argument. + * w, aligned to 16 bytes by the __int128 it holds, then leaves a stack slot + * empty before it: the fourth on x86-64, the second on arm64. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_arg_kfunc_int128_stack_hole(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_pair_arg s = { .lo = a, .hi = b }; + struct prog_test_i128_arg w = { .v = ((u128)~a << 64) | ~b }; + + if (bpf_kfunc_call_test_i128_arg_stack_hole(1, 2, 3, 4, 5, s, 6, 7, w) != + 1 + 2 * 2 + 3 * 3 + 4 * 4 + 5 * 5 + 6 * a + 7 * b + 8 * 6 + 9 * 7 + + 10 * ~b + 11 * ~a) + return 1; + + return 0; +} + /* * Both conventions pad the stack to align this __int128, and the BPF * convention pads for neither, so both JITs move it up an eightbyte. diff --git a/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c new file mode 100644 index 000000000000..5fde710c4e37 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tgt.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +__u64 base; +__u64 ret_stack; + +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +/* g and h are on the stack, for the x86-64 kernel convention too. */ +static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, __u64 h) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + h * 8; +} + +/* No stack args of its own, so it can share a program with a tail call. */ +static __noinline __u64 call_stack(__u64 x) +{ + return callee_stack(x + 1, x + 2, x + 3, x + 4, x + 5, x + 6, x + 7, + x + 8); +} + +#else + +const volatile bool has_stack_arg = false; + +#endif + +/* Makes the program tail_call_reachable, though callee_stack() is not. */ +static __noinline int subprog_tail(void *ctx) +{ + int ret = 0; + + bpf_tail_call_static(ctx, &jmp_table, 0); + barrier_var(ret); + return ret; +} + +SEC("syscall") +int run(void *ctx) +{ +#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) + ret_stack = call_stack(base); +#endif + return subprog_tail(ctx); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c new file mode 100644 index 000000000000..1800e7b7afe2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_stack_arg_tailcall_tracer.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +long stack_g, stack_h, stack_ret; + +SEC("fentry/callee_stack") +int stack_entry(unsigned long long *ctx) +{ + stack_g = ctx[6]; + stack_h = ctx[7]; + return 0; +} + +SEC("fexit/callee_stack") +int stack_exit(unsigned long long *ctx) +{ + stack_ret = ctx[8]; + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c index 4638dfec1f38..5520ee500c01 100644 --- a/tools/testing/selftests/bpf/progs/tracing_struct_int128.c +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128.c @@ -15,4 +15,49 @@ int test_int128_arg_fexit(unsigned long long *ctx) return 0; } +/* + * The arguments come packed in the BPF convention whatever holes the + * kernel convention left between them: an eightbyte each, and two for the + * struct holding an __int128. + */ +long s_g, s_h_lo, s_h_hi, s_i, s_ret; + +SEC("fentry/bpf_testmod_test_int128_stack") +int test_int128_stack_fentry(unsigned long long *ctx) +{ + s_g = ctx[6]; + s_h_lo = ctx[7]; + s_h_hi = ctx[8]; + s_i = ctx[9]; + return 0; +} + +SEC("fexit/bpf_testmod_test_int128_stack") +int test_int128_stack_fexit(unsigned long long *ctx) +{ + s_ret = ctx[10]; + return 0; +} + +long b_f_a, b_f_b, b_g, b_h, b_i_lo, b_i_hi, b_ret; + +SEC("fentry/bpf_testmod_test_int128_backfill") +int test_int128_backfill_fentry(unsigned long long *ctx) +{ + b_f_a = ctx[5]; + b_f_b = ctx[6]; + b_g = ctx[7]; + b_h = ctx[8]; + b_i_lo = ctx[9]; + b_i_hi = ctx[10]; + return 0; +} + +SEC("fexit/bpf_testmod_test_int128_backfill") +int test_int128_backfill_fexit(unsigned long long *ctx) +{ + b_ret = ctx[11]; + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c new file mode 100644 index 000000000000..2561422c25dd --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +struct int128_arg { + __int128 v; +}; + +__u64 base; +__u64 ret_odd, ret_stack, ret_split, ret_unreliable; + +#if defined(__clang__) + +const volatile bool has_struct_arg = true; + +/* arm64 would want v in x2 and x3, and b in x4. */ +__noinline __u64 callee_odd(__u64 a, struct int128_arg v, __u64 b) +{ + return a + (__u64)v.v * 2 + (__u64)((unsigned __int128)v.v >> 64) * 3 + + b * 4; +} + +#else + +const volatile bool has_struct_arg = false; + +#endif + +#if defined(__clang__) && \ + (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + +const volatile bool has_stack_arg = true; + +/* + * x86-64 would want h in its third and fourth stack slots, and arm64 in + * its first and second, leaving x7 empty. + */ +static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f, __u64 g, + struct int128_arg h, __u64 i) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + + (__u64)h.v * 8 + (__u64)((unsigned __int128)h.v >> 64) * 9 + + i * 10; +} + +/* + * x86-64 BPF code passes h split between r9 and its first stack slot; + * arm64 passes it in x5 and x6, so nothing is split there. + */ +static __noinline __u64 callee_split(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, struct int128_arg h, __u64 i) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + (__u64)h.v * 6 + + (__u64)((unsigned __int128)h.v >> 64) * 7 + i * 8; +} + +/* + * A plain pointer makes the BTF of a static function unreliable, but f is + * still passed on the stack. The store keeps p a pointer. + */ +static __noinline __u64 callee_unreliable(__u64 *p, __u64 b, __u64 c, __u64 d, + __u64 e, __u64 f) +{ + *p += 1; + return *p + b * 2 + c * 3 + d * 4 + e * 5 + f * 6; +} + +#else + +const volatile bool has_stack_arg = false; + +#endif + +SEC("syscall") +int run(void *ctx) +{ +#if defined(__clang__) + struct int128_arg v = { .v = ((__int128)(base + 9) << 64) | (base + 8) }; + + ret_odd = callee_odd(base + 1, v, base + 2); +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ + defined(__BPF_FEATURE_STACK_ARGUMENT) + ret_stack = callee_stack(base + 1, base + 2, base + 3, base + 4, + base + 5, base + 6, base + 7, v, base + 10); + ret_split = callee_split(base + 1, base + 2, base + 3, base + 4, + base + 5, v, base + 10); + { + __u64 x = base; + + ret_unreliable = callee_unreliable(&x, base + 2, base + 3, + base + 4, base + 5, base + 6); + } +#endif +#endif + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c new file mode 100644 index 000000000000..fc81c2eea4a8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tracing_struct_int128_tracer.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include + +struct int128_arg { + __int128 v; +}; + +long odd_a, odd_lo, odd_hi, odd_b, odd_ret; + +SEC("fentry/callee_odd") +int BPF_PROG2(odd_entry, __u64, a, struct int128_arg, v, __u64, b) +{ + odd_a = a; + odd_lo = (__u64)v.v; + odd_hi = (unsigned __int128)v.v >> 64; + odd_b = b; + return 0; +} + +SEC("fexit/callee_odd") +int odd_exit(unsigned long long *ctx) +{ + odd_ret = ctx[4]; + return 0; +} + +long stack_g, stack_lo, stack_hi, stack_i, stack_ret; + +SEC("fentry/callee_stack") +int BPF_PROG2(stack_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e, + __u64, f, __u64, g, struct int128_arg, h, __u64, i) +{ + stack_g = g; + stack_lo = (__u64)h.v; + stack_hi = (unsigned __int128)h.v >> 64; + stack_i = i; + return 0; +} + +SEC("fexit/callee_stack") +int stack_exit(unsigned long long *ctx) +{ + stack_ret = ctx[10]; + return 0; +} + +long split_e, split_lo, split_hi, split_i, split_ret; + +SEC("fentry/callee_split") +int BPF_PROG2(split_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e, + struct int128_arg, h, __u64, i) +{ + split_e = e; + split_lo = (__u64)h.v; + split_hi = (unsigned __int128)h.v >> 64; + split_i = i; + return 0; +} + +SEC("fexit/callee_split") +int split_exit(unsigned long long *ctx) +{ + split_ret = ctx[8]; + return 0; +} + +/* Refused: the trampoline would lose the stack argument. */ +SEC("?fentry/callee_unreliable") +int unreliable_entry(unsigned long long *ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index b4f0af2aeca4..38edb4addc5b 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -197,6 +197,43 @@ bpf_testmod_test_int128_arg(__int128 a, int b, long c) bpf_testmod_test_struct_arg_result = (long)a + b + c; return bpf_testmod_test_struct_arg_result; } + +struct bpf_testmod_struct_int128 { + __int128 a; +}; + +/* + * x86-64 puts g in the first stack slot and h, aligned to 16 bytes, in + * the third and fourth, leaving a hole. arm64 leaves x7 empty and puts h + * in the first two stack slots. Either way h and i are not where the BPF + * convention packs them, right after g. + */ +noinline long +bpf_testmod_test_int128_stack(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, + u64 g, struct bpf_testmod_struct_int128 h, u64 i) +{ + bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 + + f * 6 + g * 7 + (u64)h.a * 8 + + (u64)((unsigned __int128)h.a >> 64) * 9 + i * 10; + return bpf_testmod_test_struct_arg_result; +} + +/* + * x86-64 has no register left for f after e, so f goes to the stack and g + * takes R9 back; h goes to the third stack slot and i, aligned to 16 + * bytes, to the fifth and sixth. arm64 has registers up to g, puts h in + * the first stack slot and i in the third and fourth. + */ +noinline long +bpf_testmod_test_int128_backfill(u64 a, u64 b, u64 c, u64 d, u64 e, + struct bpf_testmod_struct_arg_2 f, u64 g, u64 h, + struct bpf_testmod_struct_int128 i) +{ + bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 + + f.a * 6 + f.b * 7 + g * 8 + h * 9 + (u64)i.a * 10 + + (u64)((unsigned __int128)i.a >> 64) * 11; + return bpf_testmod_test_struct_arg_result; +} #endif __weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void) @@ -683,6 +720,12 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, #ifdef __SIZEOF_INT128__ (void)bpf_testmod_test_int128_ret(i); (void)bpf_testmod_test_int128_arg((__int128)1, 2, 3); + (void)bpf_testmod_test_int128_stack(1, 2, 3, 4, 5, 6, 7, + (struct bpf_testmod_struct_int128){ .a = ((__int128)9 << 64) | 8 }, + 10); + (void)bpf_testmod_test_int128_backfill(1, 2, 3, 4, 5, + (struct bpf_testmod_struct_arg_2){ .a = 6, .b = 7 }, 8, 9, + (struct bpf_testmod_struct_int128){ .a = ((__int128)11 << 64) | 10 }); #endif (void)trace_bpf_testmod_test_raw_tp_null_tp(NULL); @@ -1047,6 +1090,29 @@ __bpf_kfunc u64 bpf_kfunc_call_test_i128_arg(u64 a, u64 b, __int128 v) return a + b * 2 + (u64)v * 3 + (u64)((unsigned __int128)v >> 64) * 4; } +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd(u64 a, __int128 v, u64 b) +{ + return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + b * 4; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd_many(u64 a, __int128 v, u64 b, u64 c, + u64 d, u64 e, u64 f, __int128 w) +{ + return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + + b * 4 + c * 5 + d * 6 + e * 7 + f * 8 + + (u64)w * 9 + (u64)((unsigned __int128)w >> 64) * 10; +} + +__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_stack_hole(u64 a, u64 b, u64 c, u64 d, u64 e, + struct prog_test_pair_arg s, + u64 f, u64 g, + struct prog_test_i128_arg w) +{ + return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 + + f * 8 + g * 9 + (u64)w.v * 10 + + (u64)((unsigned __int128)w.v >> 64) * 11; +} + __bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_pad(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, __int128 v) { @@ -1770,6 +1836,9 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd_many) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_stack_hole) BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_pad) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_nofit) BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_tail) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 67c02a421d13..a6a93c901416 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -66,6 +66,12 @@ struct prog_test_pair_arg { /* 16 bytes: two argument registers */ __u64 hi; }; +#ifdef __SIZEOF_INT128__ +struct prog_test_i128_arg { /* 16 bytes, aligned to 16 like its member */ + __int128 v; +}; +#endif + struct prog_test_ptr_arg { /* 16 bytes, but holds a pointer */ void *p; __u64 x; @@ -234,6 +240,13 @@ struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; __u64 bpf_kfunc_call_test_pair_arg(__u64 a, struct prog_test_pair_arg s, __u64 b) __ksym; #ifdef __SIZEOF_INT128__ __u64 bpf_kfunc_call_test_i128_arg(__u64 a, __u64 b, __int128 v) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_odd(__u64 a, __int128 v, __u64 b) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_odd_many(__u64 a, __int128 v, __u64 b, __u64 c, + __u64 d, __u64 e, __u64 f, __int128 w) __ksym; +__u64 bpf_kfunc_call_test_i128_arg_stack_hole(__u64 a, __u64 b, __u64 c, __u64 d, + __u64 e, struct prog_test_pair_arg s, + __u64 f, __u64 g, + struct prog_test_i128_arg w) __ksym; __u64 bpf_kfunc_call_test_i128_arg_pad(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __int128 v) __ksym; #endif