mirror of
https://github.com/torvalds/linux.git
synced 2026-11-07 06:17:02 +01:00
Merge branch 'bpf-follow-up-by-value-struct-and-__int128-argument-support'
Yonghong Song says:
====================
bpf: Follow up by-value struct and __int128 argument support
This is a follow-up to the series "bpf: Support by-value struct and
__int128 arguments" [1], which placed kfunc arguments by the kernel
calling convention in the x86-64 and arm64 JITs, and made the arm64
trampoline place an argument aligned to 16 bytes: an __int128, or an
aggregate holding one. This series covers what was left, and a few
problems found on the way:
- The x86-64 trampoline. x86-64 starts such an argument passed on the
stack at a 16-byte aligned slot, which can leave an empty slot
before it, but the trampoline copied the stack arguments as if
there were none. A bpf program attached to such a function read the
wrong values from the empty slot on, and with fexit, where the
trampoline calls the traced function itself, that function got its
stack arguments one slot off as well.
- A trampoline on a BPF function. BPF code calls one with argument
slot N where the kernel convention passes its Nth u64 argument, but
the trampolines placed the arguments by their types: arm64 left a
register empty before a 16-byte argument, x86-64 would not split one
between r9 and the stack, and s390 took one as a pointer to it. Such
a function is now described to its trampoline as one u64 per slot.
- The x86-64 trampoline on a function of a program that can tail call
pushed the tail call counter pointer between the stack arguments and
the return address.
- A static function whose BTF is unreliable could still take stack
arguments, while its trampoline saved and restored only five
registers. Attaching to such a function is now refused.
- Tests with an __int128 at an odd slot, where arm64 leaves a register
or a stack slot empty. On arm64 such a kfunc gets BTF only with the
pahole patch [2], now in pahole master after v1.32, so these tests
were not part of [1].
Patch 1 keeps the x86-64 trampoline's tail call counter pointer clear of
the stack arguments, and patch 2 tests it. Patch 3 adds the odd-slot
kfunc tests. Patch 4 describes a BPF function to its trampoline as one
u64 per slot, and patch 5 places the x86-64 trampoline arguments by the
calling convention. Patches 6 and 7 test trampolines on kernel functions
and on BPF functions with an argument after a 16-byte aligned one, the
latter on s390x as well. Patch 8 refuses a trampoline on an unreliable
static function with stack arguments, and patch 9 tests it, along with a
BPF function whose 16-byte argument x86-64 splits between r9 and the
stack.
[1] https://lore.kernel.org/bpf/20260912195156.980886-1-yonghong.song@linux.dev/
[2] https://lore.kernel.org/dwarves/20260917164043.1577508-1-yonghong.song@linux.dev/
Changelogs:
v3 -> v4:
- v3: https://lore.kernel.org/bpf/20261013170910.2126673-1-yonghong.song@linux.dev/
- Drop v3 patches 10 and 11 (16-byte aligned kfunc stack argument
on x86-64); no kfunc in the tree needs it.
- Drop byte-order based ctx indexing for the __int128 struct
argument; tests on BPF functions use BPF_PROG2(). (patches 6, 7
and 9)
- Run the test on BPF functions on s390x too. The functions taking
stack arguments are built only for x86 and arm64. (patch 7)
- Say that patch 4 also fixes the s390 trampoline on a BPF function
taking a 16-byte argument. (patch 4)
v2 -> v3:
- v2: https://lore.kernel.org/bpf/20261013003429.1853052-1-yonghong.song@linux.dev/
- Index the two halves of a struct holding an __int128 in ctx by the
byte order, so the tracing tests also hold on a big-endian
kernel. (patches 6, 7 and 9)
- Say that only x86-64 BPF code splits a 16-byte argument between r9
and the stack; on arm64 nothing is split. (patches 4 and 9)
v1 -> v2:
- v1: https://lore.kernel.org/bpf/20261009064206.1238047-1-yonghong.song@linux.dev/
- Replace the bpf_tgt_prog flag of v1 patch 2, which only
bpf_jit_place_args() looked at, with a model of one u64 per
argument slot for a BPF function, built in
bpf_check_attach_target(). This also fixes the arm64 trampoline,
which took a 16-byte argument split between x7 and the stack as
wholly in registers, without any arch change. (patch 4)
- Refuse a trampoline on a static function whose BTF is unreliable
but which takes stack arguments. (patches 8 and 9)
- New: keep the x86-64 trampoline's tail call counter pointer in a
slot of its frame instead of pushing it between the stack
arguments and the return address. (patches 1 and 2)
- New: pass a kfunc's 16-byte aligned stack argument 16-byte aligned
in the x86-64 JIT, with bpf_jit_stack_slots_to_align() and
bpf_jit_plan_slot_moves() in kernel/bpf/core.c. (patches 10 and
11)
- Reword the x86-64 placement patch and the BPF function test for
the new model. (patches 5 and 7)
====================
Link: https://patch.msgid.link/20261016183039.1132126-1-yonghong.song@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov
commit
645f5fcf7a
|
|
@ -2359,6 +2359,16 @@ static const u8 x86_arg_reg[] = {
|
|||
BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5, X86_REG_R9,
|
||||
};
|
||||
|
||||
static bool x86_arg_on_stack(u8 pos)
|
||||
{
|
||||
return pos >= x86_arg_abi.nr_arg_regs;
|
||||
}
|
||||
|
||||
static s32 x86_arg_stack_off(u8 pos)
|
||||
{
|
||||
return (pos - x86_arg_abi.nr_arg_regs) * 8;
|
||||
}
|
||||
|
||||
/*
|
||||
* Move the arguments the x86-64 ABI places somewhere other than the argument
|
||||
* slot the BPF calling convention gave them. @stack_base addresses the
|
||||
|
|
@ -3883,21 +3893,17 @@ static void clean_stack_garbage(const struct btf_func_model *m,
|
|||
}
|
||||
}
|
||||
|
||||
/* get the count of the regs that are used to pass arguments */
|
||||
static int get_nr_used_regs(const struct btf_func_model *m)
|
||||
/* the eightbytes taken by the on-stack arguments, alignment holes included */
|
||||
static int get_nr_stack_slots(const u8 *pos_of_slot, int nr_slots)
|
||||
{
|
||||
int i, arg_regs, nr_used_regs = 0;
|
||||
int slot;
|
||||
|
||||
for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) {
|
||||
arg_regs = (m->arg_size[i] + 7) / 8;
|
||||
if (nr_used_regs + arg_regs <= 6)
|
||||
nr_used_regs += arg_regs;
|
||||
|
||||
if (nr_used_regs >= 6)
|
||||
break;
|
||||
for (slot = nr_slots - 1; slot >= 0; slot--) {
|
||||
if (x86_arg_on_stack(pos_of_slot[slot]))
|
||||
return x86_arg_stack_off(pos_of_slot[slot]) / 8 + 1;
|
||||
}
|
||||
|
||||
return nr_used_regs;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -3927,20 +3933,21 @@ static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base
|
|||
}
|
||||
|
||||
static void save_args(const struct btf_func_model *m, u8 **prog,
|
||||
int stack_size, bool for_call_origin, u32 flags,
|
||||
u64 arena_base)
|
||||
const u8 *pos_of_slot, int stack_size,
|
||||
bool for_call_origin, u32 flags, u64 arena_base)
|
||||
{
|
||||
int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0;
|
||||
int arg_regs, first_off = 0, nr_stack_slots = 0;
|
||||
bool use_jmp = bpf_trampoline_use_jmp(flags);
|
||||
int stack_args_off = (use_jmp || (flags & BPF_TRAMP_F_INDIRECT)) ? 16 : 24;
|
||||
int i, j;
|
||||
int bargs_off = stack_size;
|
||||
int i, j, slot, off, doff;
|
||||
|
||||
/* Store function arguments to stack.
|
||||
* For a function that accepts two pointers the sequence will be:
|
||||
* mov QWORD PTR [rbp-0x10],rdi
|
||||
* mov QWORD PTR [rbp-0x8],rsi
|
||||
*/
|
||||
for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) {
|
||||
for (i = 0, slot = 0; i < m->nr_args; i++) {
|
||||
bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG);
|
||||
bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG;
|
||||
|
||||
|
|
@ -3962,8 +3969,36 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
|
|||
*
|
||||
* the arg1-5,arg7 will be passed by regs, and arg6 will
|
||||
* by stack.
|
||||
*
|
||||
* An argument aligned to 16 bytes, an __int128 or a struct
|
||||
* holding one, also starts at an even stack slot, which can
|
||||
* leave a hole before it. bpf_jit_place_args() worked out
|
||||
* where each eightbyte is.
|
||||
*/
|
||||
if (nr_regs + arg_regs > 6) {
|
||||
for (j = 0; j < arg_regs; j++, slot++, bargs_off -= 8) {
|
||||
u8 pos = pos_of_slot[slot];
|
||||
u32 src;
|
||||
|
||||
if (!x86_arg_on_stack(pos)) {
|
||||
/*
|
||||
* Only copy the arguments on-stack to
|
||||
* current 'stack_size' and ignore the regs,
|
||||
* used to prepare the arguments on-stack for
|
||||
* origin call.
|
||||
*/
|
||||
if (for_call_origin)
|
||||
continue;
|
||||
|
||||
/* copy the arguments from regs into stack */
|
||||
src = x86_arg_reg[pos];
|
||||
if (arena_arg) {
|
||||
emit_arena_arg_conv(prog, src, nullable, (u32)arena_base);
|
||||
src = BPF_REG_0;
|
||||
}
|
||||
emit_stx(prog, BPF_DW, BPF_REG_FP, src, -bargs_off);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* copy function arguments from origin stack frame
|
||||
* into current stack frame.
|
||||
*
|
||||
|
|
@ -3973,53 +4008,33 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
|
|||
* entered through the fentry call, so rbp + 24, and
|
||||
* a single one when it is entered with a jmp or
|
||||
* called indirectly, so rbp + 16.
|
||||
*
|
||||
* The area for the origin call mirrors the incoming
|
||||
* one, hole and all; only the bpf program takes the
|
||||
* arguments packed.
|
||||
*/
|
||||
for (j = 0; j < arg_regs; j++) {
|
||||
emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP,
|
||||
nr_stack_slots * 8 + stack_args_off);
|
||||
if (arena_arg)
|
||||
emit_arena_arg_conv(prog, BPF_REG_0, nullable,
|
||||
(u32)arena_base);
|
||||
emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0,
|
||||
-stack_size);
|
||||
off = x86_arg_stack_off(pos);
|
||||
doff = for_call_origin ? stack_size - off : bargs_off;
|
||||
emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP,
|
||||
stack_args_off + off);
|
||||
if (arena_arg)
|
||||
emit_arena_arg_conv(prog, BPF_REG_0, nullable,
|
||||
(u32)arena_base);
|
||||
emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -doff);
|
||||
|
||||
if (!nr_stack_slots)
|
||||
first_off = stack_size;
|
||||
stack_size -= 8;
|
||||
nr_stack_slots++;
|
||||
}
|
||||
} else {
|
||||
/* Only copy the arguments on-stack to current
|
||||
* 'stack_size' and ignore the regs, used to
|
||||
* prepare the arguments on-stack for origin call.
|
||||
*/
|
||||
if (for_call_origin) {
|
||||
nr_regs += arg_regs;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* copy the arguments from regs into stack */
|
||||
for (j = 0; j < arg_regs; j++) {
|
||||
u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs;
|
||||
|
||||
if (arena_arg) {
|
||||
emit_arena_arg_conv(prog, src, nullable, (u32)arena_base);
|
||||
src = BPF_REG_0;
|
||||
}
|
||||
emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size);
|
||||
stack_size -= 8;
|
||||
nr_regs++;
|
||||
}
|
||||
if (!nr_stack_slots)
|
||||
first_off = doff;
|
||||
nr_stack_slots++;
|
||||
}
|
||||
}
|
||||
|
||||
clean_stack_garbage(m, prog, nr_stack_slots, first_off);
|
||||
}
|
||||
|
||||
static void restore_regs(const struct btf_func_model *m, u8 **prog,
|
||||
static void restore_regs(u8 **prog, const u8 *pos_of_slot, int nr_slots,
|
||||
int stack_size)
|
||||
{
|
||||
int i, j, arg_regs, nr_regs = 0;
|
||||
int slot;
|
||||
|
||||
/* Restore function arguments from stack.
|
||||
* For a function that accepts two pointers the sequence will be:
|
||||
|
|
@ -4028,23 +4043,10 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog,
|
|||
*
|
||||
* The logic here is similar to what we do in save_args()
|
||||
*/
|
||||
for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) {
|
||||
arg_regs = (m->arg_size[i] + 7) / 8;
|
||||
if (nr_regs + arg_regs <= 6) {
|
||||
for (j = 0; j < arg_regs; j++) {
|
||||
emit_ldx(prog, BPF_DW,
|
||||
nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs,
|
||||
BPF_REG_FP,
|
||||
-stack_size);
|
||||
stack_size -= 8;
|
||||
nr_regs++;
|
||||
}
|
||||
} else {
|
||||
stack_size -= 8 * arg_regs;
|
||||
}
|
||||
|
||||
if (nr_regs >= 6)
|
||||
break;
|
||||
for (slot = 0; slot < nr_slots; slot++, stack_size -= 8) {
|
||||
if (!x86_arg_on_stack(pos_of_slot[slot]))
|
||||
emit_ldx(prog, BPF_DW, x86_arg_reg[pos_of_slot[slot]],
|
||||
BPF_REG_FP, -stack_size);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -4233,9 +4235,9 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* mov rax, qword ptr [rbp - rounded_stack_depth - 8] */
|
||||
#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack) \
|
||||
__LOAD_TCC_PTR(-round_up(stack, 8) - 8)
|
||||
/* mov rax, qword ptr [rbp - off] */
|
||||
#define LOAD_TRAMP_TAIL_CALL_CNT_PTR(off) \
|
||||
__LOAD_TCC_PTR(-(off))
|
||||
|
||||
/* Example:
|
||||
* __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev);
|
||||
|
|
@ -4303,8 +4305,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
struct bpf_tramp_nodes *tnodes,
|
||||
void *func_addr)
|
||||
{
|
||||
int i, ret, nr_regs = m->nr_args, stack_size = 0;
|
||||
int i, ret, nr_regs, stack_size = 0;
|
||||
u8 pos_of_slot[MAX_BPF_FUNC_ARG_SLOTS];
|
||||
int regs_off, func_meta_off, ip_off, run_ctx_off, arg_stack_off, rbx_off;
|
||||
int tcc_off = 0;
|
||||
struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY];
|
||||
struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT];
|
||||
struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN];
|
||||
|
|
@ -4326,12 +4330,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
|
||||
arena_base = bpf_tramp_arena_base(m, tnodes, flags);
|
||||
|
||||
for (i = 0; i < m->nr_args; i++)
|
||||
nr_regs += (m->arg_size[i] + 7) / 8 - 1;
|
||||
|
||||
/* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6
|
||||
* are passed through regs, the remains are through stack.
|
||||
/*
|
||||
* x86-64 supports up to MAX_BPF_FUNC_ARGS argument slots, six in
|
||||
* registers and the rest on the stack, where bpf_jit_place_args()
|
||||
* puts them.
|
||||
*/
|
||||
nr_regs = bpf_jit_place_args(&x86_arg_abi, m, pos_of_slot);
|
||||
if (nr_regs > MAX_BPF_FUNC_ARGS)
|
||||
return -ENOTSUPP;
|
||||
|
||||
|
|
@ -4343,11 +4347,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
* RBP - 8 [ return value ] BPF_TRAMP_F_CALL_ORIG or
|
||||
* BPF_TRAMP_F_RET_FENTRY_RET flags
|
||||
*
|
||||
* [ reg_argN ] always
|
||||
* [ ... ]
|
||||
* RBP - regs_off [ reg_arg1 ] program's ctx pointer
|
||||
* [ arg slotN ] always, packed, whether
|
||||
* [ ... ] in a register or on the
|
||||
* RBP - regs_off [ arg slot1 ] stack; program's ctx pointer
|
||||
*
|
||||
* RBP - func_meta_off [ regs count, etc ] always
|
||||
* RBP - func_meta_off [ slot count, etc ] always
|
||||
*
|
||||
* RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag
|
||||
*
|
||||
|
|
@ -4355,11 +4359,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
*
|
||||
* RBP - run_ctx_off [ bpf_tramp_run_ctx ]
|
||||
*
|
||||
* [ stack_argN ] BPF_TRAMP_F_CALL_ORIG
|
||||
* [ ... ]
|
||||
* [ stack_arg2 ]
|
||||
* RBP - arg_stack_off [ stack_arg1 ]
|
||||
* RSP [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX
|
||||
* RBP - tcc_off [ tail_call_cnt_ptr ] BPF_TRAMP_F_TAIL_CALL_CTX
|
||||
*
|
||||
* [ stack slotN ] BPF_TRAMP_F_CALL_ORIG,
|
||||
* [ ... ] the incoming stack
|
||||
* [ stack slot2 ] arguments, alignment
|
||||
* RBP - arg_stack_off [ stack slot1 ] holes included
|
||||
*/
|
||||
|
||||
/* room for return value of orig_call or fentry prog */
|
||||
|
|
@ -4370,7 +4375,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
stack_size += nr_regs * 8;
|
||||
regs_off = stack_size;
|
||||
|
||||
/* function matedata, such as regs count */
|
||||
/* function matedata, such as the argument slot count */
|
||||
stack_size += 8;
|
||||
func_meta_off = stack_size;
|
||||
|
||||
|
|
@ -4390,9 +4395,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7;
|
||||
run_ctx_off = stack_size;
|
||||
|
||||
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
|
||||
stack_size += 8;
|
||||
tcc_off = stack_size;
|
||||
}
|
||||
|
||||
if (nr_regs > 6 && (flags & BPF_TRAMP_F_CALL_ORIG)) {
|
||||
/* the space that used to pass arguments on-stack */
|
||||
stack_size += (nr_regs - get_nr_used_regs(m)) * 8;
|
||||
stack_size += get_nr_stack_slots(pos_of_slot, nr_regs) * 8;
|
||||
/* make sure the stack pointer is 16-byte aligned if we
|
||||
* need pass arguments on stack, which means
|
||||
* [stack_size + 8(rbp) + 8(rip) + 8(origin rip)]
|
||||
|
|
@ -4442,12 +4452,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
/* sub rsp, stack_size */
|
||||
emit_sub_rsp(&prog, stack_size);
|
||||
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX)
|
||||
EMIT1(0x50); /* push rax */
|
||||
/* mov QWORD PTR [rbp - tcc_off], rax */
|
||||
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -tcc_off);
|
||||
/* mov QWORD PTR [rbp - rbx_off], rbx */
|
||||
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off);
|
||||
|
||||
func_meta = nr_regs;
|
||||
/* Store number of argument registers of the traced function */
|
||||
/* Store number of argument slots of the traced function */
|
||||
emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta);
|
||||
|
||||
if (flags & BPF_TRAMP_F_IP_ARG) {
|
||||
|
|
@ -4455,7 +4466,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr);
|
||||
}
|
||||
|
||||
save_args(m, &prog, regs_off, false, flags, arena_base);
|
||||
save_args(m, &prog, pos_of_slot, regs_off, false, flags, arena_base);
|
||||
|
||||
if (flags & BPF_TRAMP_F_CALL_ORIG) {
|
||||
/* arg1: mov rdi, im */
|
||||
|
|
@ -4496,14 +4507,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
}
|
||||
|
||||
if (flags & BPF_TRAMP_F_CALL_ORIG) {
|
||||
restore_regs(m, &prog, regs_off);
|
||||
save_args(m, &prog, arg_stack_off, true, flags, 0);
|
||||
restore_regs(&prog, pos_of_slot, nr_regs, regs_off);
|
||||
save_args(m, &prog, pos_of_slot, arg_stack_off, true, flags, 0);
|
||||
|
||||
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
|
||||
/* Before calling the original function, load the
|
||||
* tail_call_cnt_ptr from stack to rax.
|
||||
*/
|
||||
LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size);
|
||||
LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off);
|
||||
}
|
||||
|
||||
if (flags & BPF_TRAMP_F_ORIG_STACK) {
|
||||
|
|
@ -4550,7 +4561,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
}
|
||||
|
||||
if (flags & BPF_TRAMP_F_RESTORE_REGS)
|
||||
restore_regs(m, &prog, regs_off);
|
||||
restore_regs(&prog, pos_of_slot, nr_regs, regs_off);
|
||||
|
||||
/* This needs to be done regardless. If there were fmod_ret programs,
|
||||
* the return value is only updated on the stack and still needs to be
|
||||
|
|
@ -4567,7 +4578,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
|
|||
/* Before running the original function, load the
|
||||
* tail_call_cnt_ptr from stack to rax.
|
||||
*/
|
||||
LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size);
|
||||
LOAD_TRAMP_TAIL_CALL_CNT_PTR(tcc_off);
|
||||
}
|
||||
|
||||
/* restore return value of orig_call or fentry prog back into RAX */
|
||||
|
|
|
|||
|
|
@ -22006,6 +22006,59 @@ static bool attach_uses_trampoline_retval(enum bpf_attach_type type)
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* BPF code passes argument slot N where the kernel convention passes its Nth
|
||||
* u64 argument, so describe a BPF function to its trampoline as one u64 per
|
||||
* slot, and every arch places its arguments right.
|
||||
*/
|
||||
static int flatten_bpf_func_model(struct bpf_verifier_log *log, const char *tname,
|
||||
struct btf_func_model *m)
|
||||
{
|
||||
u32 i, nslots = 0;
|
||||
|
||||
for (i = 0; i < m->nr_args; i++)
|
||||
nslots += btf_func_model_arg_slots(m, i);
|
||||
if (nslots > MAX_BPF_FUNC_ARGS) {
|
||||
bpf_log(log, "The function %s has %u argument slots. Too many.\n",
|
||||
tname, nslots);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
for (i = 0; i < nslots; i++) {
|
||||
m->arg_size[i] = sizeof(u64);
|
||||
m->arg_flags[i] = 0;
|
||||
}
|
||||
m->nr_args = nslots;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* The verifier lets a static function use stack arguments counted from its
|
||||
* BTF, even if that BTF turns out unreliable. A trampoline on the function is
|
||||
* then given only MAX_BPF_FUNC_REG_ARGS u64 arguments and would lose the
|
||||
* stack arguments, so such a function cannot be attached to.
|
||||
*/
|
||||
static int check_unreliable_func_args(struct bpf_verifier_log *log, struct btf *btf,
|
||||
const struct btf_type *func_proto, const char *tname)
|
||||
{
|
||||
struct btf_func_model m;
|
||||
u32 i, nslots = 0;
|
||||
|
||||
if (btf_distill_func_proto(NULL, btf, func_proto, tname, &m)) {
|
||||
/* at least a slot for each argument */
|
||||
nslots = btf_type_vlen(func_proto);
|
||||
} else {
|
||||
for (i = 0; i < m.nr_args; i++)
|
||||
nslots += btf_func_model_arg_slots(&m, i);
|
||||
}
|
||||
if (nslots <= MAX_BPF_FUNC_REG_ARGS)
|
||||
return 0;
|
||||
|
||||
bpf_log(log, "Cannot attach to static function %s: unreliable BTF and stack arguments\n",
|
||||
tname);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
int bpf_check_attach_target(struct bpf_verifier_log *log,
|
||||
const struct bpf_prog *prog,
|
||||
const struct bpf_prog *tgt_prog,
|
||||
|
|
@ -22273,12 +22326,23 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
|
|||
prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
|
||||
return -EINVAL;
|
||||
|
||||
if (tgt_prog && conservative)
|
||||
if (tgt_prog && conservative) {
|
||||
if (subprog > 0) {
|
||||
ret = check_unreliable_func_args(log, btf, t, tname);
|
||||
if (ret)
|
||||
return ret;
|
||||
}
|
||||
t = NULL;
|
||||
}
|
||||
|
||||
ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
if (tgt_prog) {
|
||||
ret = flatten_bpf_func_model(log, tname, &tgt_info->fmodel);
|
||||
if (ret < 0)
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (tgt_info->fmodel.ret_size > 8 &&
|
||||
attach_uses_trampoline_retval(prog->expected_attach_type)) {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,10 @@
|
|||
#include "tracing_struct.skel.h"
|
||||
#include "tracing_struct_many_args.skel.h"
|
||||
#include "tracing_struct_int128.skel.h"
|
||||
#include "tracing_struct_int128_tgt.skel.h"
|
||||
#include "tracing_struct_int128_tracer.skel.h"
|
||||
#include "tracing_stack_arg_tailcall_tgt.skel.h"
|
||||
#include "tracing_stack_arg_tailcall_tracer.skel.h"
|
||||
|
||||
static void test_struct_args(void)
|
||||
{
|
||||
|
|
@ -117,9 +121,11 @@ static void test_int128_args(void)
|
|||
{
|
||||
/*
|
||||
* __int128 arguments are passed in a register pair on x86_64 and
|
||||
* arm64, which the trampoline packs into two context slots. Other
|
||||
* architectures pass a __int128 differently (e.g. s390x passes larger
|
||||
* arguments by reference), so only exercise this on x86_64 and arm64.
|
||||
* arm64, which the trampoline packs into two context slots. Past the
|
||||
* registers, both start an argument aligned to 16 bytes at an even
|
||||
* stack slot, which the trampoline packs too. Other architectures
|
||||
* pass a __int128 differently (e.g. s390x passes larger arguments by
|
||||
* reference), so only exercise this on x86_64 and arm64.
|
||||
*/
|
||||
#if defined(__x86_64__) || defined(__aarch64__)
|
||||
struct tracing_struct_int128 *skel;
|
||||
|
|
@ -139,6 +145,20 @@ static void test_int128_args(void)
|
|||
ASSERT_EQ(skel->bss->t_c, 3, "t:c");
|
||||
ASSERT_EQ(skel->bss->t_ret, 6, "t ret");
|
||||
|
||||
ASSERT_EQ(skel->bss->s_g, 7, "s:g");
|
||||
ASSERT_EQ(skel->bss->s_h_lo, 8, "s:h.lo");
|
||||
ASSERT_EQ(skel->bss->s_h_hi, 9, "s:h.hi");
|
||||
ASSERT_EQ(skel->bss->s_i, 10, "s:i");
|
||||
ASSERT_EQ(skel->bss->s_ret, 385, "s ret");
|
||||
|
||||
ASSERT_EQ(skel->bss->b_f_a, 6, "b:f.a");
|
||||
ASSERT_EQ(skel->bss->b_f_b, 7, "b:f.b");
|
||||
ASSERT_EQ(skel->bss->b_g, 8, "b:g");
|
||||
ASSERT_EQ(skel->bss->b_h, 9, "b:h");
|
||||
ASSERT_EQ(skel->bss->b_i_lo, 10, "b:i.lo");
|
||||
ASSERT_EQ(skel->bss->b_i_hi, 11, "b:i.hi");
|
||||
ASSERT_EQ(skel->bss->b_ret, 506, "b ret");
|
||||
|
||||
destroy_skel:
|
||||
tracing_struct_int128__destroy(skel);
|
||||
#else
|
||||
|
|
@ -146,6 +166,191 @@ static void test_int128_args(void)
|
|||
#endif
|
||||
}
|
||||
|
||||
static void test_int128_tgt_prog(void)
|
||||
{
|
||||
#if defined(__x86_64__) || defined(__aarch64__) || defined(__s390x__)
|
||||
struct tracing_struct_int128_tracer *tracer = NULL;
|
||||
struct tracing_struct_int128_tgt *tgt;
|
||||
LIBBPF_OPTS(bpf_test_run_opts, topts);
|
||||
int err, fd;
|
||||
|
||||
tgt = tracing_struct_int128_tgt__open_and_load();
|
||||
if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load"))
|
||||
return;
|
||||
if (!tgt->rodata->has_struct_arg) {
|
||||
test__skip();
|
||||
goto out;
|
||||
}
|
||||
fd = bpf_program__fd(tgt->progs.run);
|
||||
|
||||
tracer = tracing_struct_int128_tracer__open();
|
||||
if (!ASSERT_OK_PTR(tracer, "tracer__open"))
|
||||
goto out;
|
||||
|
||||
err = bpf_program__set_attach_target(tracer->progs.odd_entry, fd, "callee_odd");
|
||||
err = err ?: bpf_program__set_attach_target(tracer->progs.odd_exit, fd, "callee_odd");
|
||||
if (tgt->rodata->has_stack_arg) {
|
||||
err = err ?: bpf_program__set_attach_target(tracer->progs.stack_entry, fd,
|
||||
"callee_stack");
|
||||
err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd,
|
||||
"callee_stack");
|
||||
err = err ?: bpf_program__set_attach_target(tracer->progs.split_entry, fd,
|
||||
"callee_split");
|
||||
err = err ?: bpf_program__set_attach_target(tracer->progs.split_exit, fd,
|
||||
"callee_split");
|
||||
} else {
|
||||
bpf_program__set_autoload(tracer->progs.stack_entry, false);
|
||||
bpf_program__set_autoload(tracer->progs.stack_exit, false);
|
||||
bpf_program__set_autoload(tracer->progs.split_entry, false);
|
||||
bpf_program__set_autoload(tracer->progs.split_exit, false);
|
||||
}
|
||||
if (!ASSERT_OK(err, "set_attach_target"))
|
||||
goto out;
|
||||
|
||||
err = tracing_struct_int128_tracer__load(tracer);
|
||||
if (!ASSERT_OK(err, "tracer__load"))
|
||||
goto out;
|
||||
err = tracing_struct_int128_tracer__attach(tracer);
|
||||
if (!ASSERT_OK(err, "tracer__attach"))
|
||||
goto out;
|
||||
|
||||
err = bpf_prog_test_run_opts(fd, &topts);
|
||||
if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval"))
|
||||
goto out;
|
||||
|
||||
ASSERT_EQ(tracer->bss->odd_a, 1, "odd:a");
|
||||
ASSERT_EQ(tracer->bss->odd_lo, 8, "odd:v.lo");
|
||||
ASSERT_EQ(tracer->bss->odd_hi, 9, "odd:v.hi");
|
||||
ASSERT_EQ(tracer->bss->odd_b, 2, "odd:b");
|
||||
ASSERT_EQ(tgt->bss->ret_odd, 52, "odd ret");
|
||||
ASSERT_EQ(tracer->bss->odd_ret, 52, "odd fexit ret");
|
||||
|
||||
if (tgt->rodata->has_stack_arg) {
|
||||
ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g");
|
||||
ASSERT_EQ(tracer->bss->stack_lo, 8, "stack:h.lo");
|
||||
ASSERT_EQ(tracer->bss->stack_hi, 9, "stack:h.hi");
|
||||
ASSERT_EQ(tracer->bss->stack_i, 10, "stack:i");
|
||||
ASSERT_EQ(tgt->bss->ret_stack, 385, "stack ret");
|
||||
ASSERT_EQ(tracer->bss->stack_ret, 385, "stack fexit ret");
|
||||
|
||||
ASSERT_EQ(tracer->bss->split_e, 5, "split:e");
|
||||
ASSERT_EQ(tracer->bss->split_lo, 8, "split:h.lo");
|
||||
ASSERT_EQ(tracer->bss->split_hi, 9, "split:h.hi");
|
||||
ASSERT_EQ(tracer->bss->split_i, 10, "split:i");
|
||||
ASSERT_EQ(tgt->bss->ret_split, 246, "split ret");
|
||||
ASSERT_EQ(tracer->bss->split_ret, 246, "split fexit ret");
|
||||
}
|
||||
|
||||
out:
|
||||
tracing_struct_int128_tracer__destroy(tracer);
|
||||
tracing_struct_int128_tgt__destroy(tgt);
|
||||
#else
|
||||
test__skip();
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* A static function whose BTF is unreliable gets a trampoline that only knows
|
||||
* of the register arguments, so one taking stack arguments must be refused.
|
||||
*/
|
||||
static void test_tgt_prog_unreliable(void)
|
||||
{
|
||||
#if defined(__x86_64__) || defined(__aarch64__)
|
||||
struct tracing_struct_int128_tracer *tracer = NULL;
|
||||
struct tracing_struct_int128_tgt *tgt;
|
||||
char log_buf[4096] = {};
|
||||
struct bpf_program *prog;
|
||||
int err, fd;
|
||||
|
||||
tgt = tracing_struct_int128_tgt__open_and_load();
|
||||
if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load"))
|
||||
return;
|
||||
if (!tgt->rodata->has_stack_arg) {
|
||||
test__skip();
|
||||
goto out;
|
||||
}
|
||||
fd = bpf_program__fd(tgt->progs.run);
|
||||
|
||||
tracer = tracing_struct_int128_tracer__open();
|
||||
if (!ASSERT_OK_PTR(tracer, "tracer__open"))
|
||||
goto out;
|
||||
|
||||
bpf_object__for_each_program(prog, tracer->obj)
|
||||
bpf_program__set_autoload(prog, false);
|
||||
bpf_program__set_autoload(tracer->progs.unreliable_entry, true);
|
||||
bpf_program__set_log_buf(tracer->progs.unreliable_entry, log_buf, sizeof(log_buf));
|
||||
err = bpf_program__set_attach_target(tracer->progs.unreliable_entry, fd,
|
||||
"callee_unreliable");
|
||||
if (!ASSERT_OK(err, "set_attach_target"))
|
||||
goto out;
|
||||
|
||||
err = tracing_struct_int128_tracer__load(tracer);
|
||||
ASSERT_ERR(err, "tracer__load");
|
||||
ASSERT_HAS_SUBSTR(log_buf, "unreliable BTF and stack arguments", "log");
|
||||
|
||||
out:
|
||||
tracing_struct_int128_tracer__destroy(tracer);
|
||||
tracing_struct_int128_tgt__destroy(tgt);
|
||||
#else
|
||||
test__skip();
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* A trampoline on a function of a tail_call_reachable program keeps the tail
|
||||
* call counter pointer on its stack, and has to hand the function its stack
|
||||
* arguments past it.
|
||||
*/
|
||||
static void test_stack_arg_tailcall(void)
|
||||
{
|
||||
#if defined(__x86_64__) || defined(__aarch64__)
|
||||
struct tracing_stack_arg_tailcall_tracer *tracer = NULL;
|
||||
struct tracing_stack_arg_tailcall_tgt *tgt;
|
||||
LIBBPF_OPTS(bpf_test_run_opts, topts);
|
||||
int err, fd;
|
||||
|
||||
tgt = tracing_stack_arg_tailcall_tgt__open_and_load();
|
||||
if (!ASSERT_OK_PTR(tgt, "tgt__open_and_load"))
|
||||
return;
|
||||
if (!tgt->rodata->has_stack_arg) {
|
||||
test__skip();
|
||||
goto out;
|
||||
}
|
||||
fd = bpf_program__fd(tgt->progs.run);
|
||||
|
||||
tracer = tracing_stack_arg_tailcall_tracer__open();
|
||||
if (!ASSERT_OK_PTR(tracer, "tracer__open"))
|
||||
goto out;
|
||||
|
||||
err = bpf_program__set_attach_target(tracer->progs.stack_entry, fd, "callee_stack");
|
||||
err = err ?: bpf_program__set_attach_target(tracer->progs.stack_exit, fd, "callee_stack");
|
||||
if (!ASSERT_OK(err, "set_attach_target"))
|
||||
goto out;
|
||||
|
||||
err = tracing_stack_arg_tailcall_tracer__load(tracer);
|
||||
if (!ASSERT_OK(err, "tracer__load"))
|
||||
goto out;
|
||||
err = tracing_stack_arg_tailcall_tracer__attach(tracer);
|
||||
if (!ASSERT_OK(err, "tracer__attach"))
|
||||
goto out;
|
||||
|
||||
err = bpf_prog_test_run_opts(fd, &topts);
|
||||
if (!ASSERT_OK(err, "test_run") || !ASSERT_EQ(topts.retval, 0, "retval"))
|
||||
goto out;
|
||||
|
||||
ASSERT_EQ(tracer->bss->stack_g, 7, "stack:g");
|
||||
ASSERT_EQ(tracer->bss->stack_h, 8, "stack:h");
|
||||
ASSERT_EQ(tgt->bss->ret_stack, 204, "stack ret");
|
||||
ASSERT_EQ(tracer->bss->stack_ret, 204, "stack fexit ret");
|
||||
|
||||
out:
|
||||
tracing_stack_arg_tailcall_tracer__destroy(tracer);
|
||||
tracing_stack_arg_tailcall_tgt__destroy(tgt);
|
||||
#else
|
||||
test__skip();
|
||||
#endif
|
||||
}
|
||||
|
||||
static void test_union_args(void)
|
||||
{
|
||||
struct tracing_struct *skel;
|
||||
|
|
@ -181,6 +386,12 @@ void test_tracing_struct(void)
|
|||
test_struct_many_args();
|
||||
if (test__start_subtest("int128_args"))
|
||||
test_int128_args();
|
||||
if (test__start_subtest("int128_tgt_prog"))
|
||||
test_int128_tgt_prog();
|
||||
if (test__start_subtest("tgt_prog_unreliable"))
|
||||
test_tgt_prog_unreliable();
|
||||
if (test__start_subtest("stack_arg_tailcall"))
|
||||
test_stack_arg_tailcall();
|
||||
if (test__start_subtest("union_args"))
|
||||
test_union_args();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,6 +50,27 @@ int aggregate_arg_kfunc_int128(struct __sk_buff *skb)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* arm64 rounds the register number up to an even one for an argument
|
||||
* aligned to 16 bytes, so it wants this __int128 in x2 and x3, leaving x1
|
||||
* empty, and the last argument in x4. The x86-64 ABI has no such rule.
|
||||
*/
|
||||
SEC("tc")
|
||||
__arch_x86_64 __arch_arm64
|
||||
__load_if_JITed()
|
||||
__success __retval(0)
|
||||
int aggregate_arg_kfunc_int128_odd(struct __sk_buff *skb)
|
||||
{
|
||||
__u64 a = skb->len ^ MIX_A;
|
||||
__u64 b = skb->len ^ MIX_B;
|
||||
u128 v = ((u128)a << 64) | b;
|
||||
|
||||
if (bpf_kfunc_call_test_i128_arg_odd(1, v, 2) != 1 + 2 * b + 3 * a + 4 * 2)
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#endif /* __SIZEOF_INT128__ */
|
||||
|
||||
#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT)
|
||||
|
|
@ -131,6 +152,56 @@ int aggregate_arg_kfunc_split8(struct __sk_buff *skb)
|
|||
|
||||
#ifdef __SIZEOF_INT128__
|
||||
|
||||
/*
|
||||
* The hole of aggregate_arg_kfunc_int128_odd, with arguments after it that
|
||||
* arm64 shifts through x5 to x7, where its JIT keeps the BPF stack slots,
|
||||
* and onto its stack. The second __int128 then leaves the second arm64
|
||||
* stack slot empty. x86-64 packs the eightbytes like the BPF convention,
|
||||
* so nothing moves there.
|
||||
*/
|
||||
SEC("tc")
|
||||
__arch_x86_64 __arch_arm64
|
||||
__load_if_JITed()
|
||||
__success __retval(0)
|
||||
int aggregate_arg_kfunc_int128_odd_many(struct __sk_buff *skb)
|
||||
{
|
||||
__u64 a = skb->len ^ MIX_A;
|
||||
__u64 b = skb->len ^ MIX_B;
|
||||
u128 v = ((u128)a << 64) | b;
|
||||
u128 w = ((u128)~a << 64) | ~b;
|
||||
|
||||
if (bpf_kfunc_call_test_i128_arg_odd_many(1, v, 2, 3, 4, 5, 6, w) !=
|
||||
1 + 2 * b + 3 * a + 4 * 2 + 5 * 3 + 6 * 4 + 7 * 5 + 8 * 6 +
|
||||
9 * ~b + 10 * ~a)
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* x86-64 moves s to the stack and gives R9 back to the seventh argument.
|
||||
* w, aligned to 16 bytes by the __int128 it holds, then leaves a stack slot
|
||||
* empty before it: the fourth on x86-64, the second on arm64.
|
||||
*/
|
||||
SEC("tc")
|
||||
__arch_x86_64 __arch_arm64
|
||||
__load_if_JITed()
|
||||
__success __retval(0)
|
||||
int aggregate_arg_kfunc_int128_stack_hole(struct __sk_buff *skb)
|
||||
{
|
||||
__u64 a = skb->len ^ MIX_A;
|
||||
__u64 b = skb->len ^ MIX_B;
|
||||
struct prog_test_pair_arg s = { .lo = a, .hi = b };
|
||||
struct prog_test_i128_arg w = { .v = ((u128)~a << 64) | ~b };
|
||||
|
||||
if (bpf_kfunc_call_test_i128_arg_stack_hole(1, 2, 3, 4, 5, s, 6, 7, w) !=
|
||||
1 + 2 * 2 + 3 * 3 + 4 * 4 + 5 * 5 + 6 * a + 7 * b + 8 * 6 + 9 * 7 +
|
||||
10 * ~b + 11 * ~a)
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Both conventions pad the stack to align this __int128, and the BPF
|
||||
* convention pads for neither, so both JITs move it up an eightbyte.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
|
||||
#include <vmlinux.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
|
||||
__uint(max_entries, 1);
|
||||
__uint(key_size, sizeof(__u32));
|
||||
__uint(value_size, sizeof(__u32));
|
||||
} jmp_table SEC(".maps");
|
||||
|
||||
__u64 base;
|
||||
__u64 ret_stack;
|
||||
|
||||
#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT)
|
||||
|
||||
const volatile bool has_stack_arg = true;
|
||||
|
||||
/* g and h are on the stack, for the x86-64 kernel convention too. */
|
||||
static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d,
|
||||
__u64 e, __u64 f, __u64 g, __u64 h)
|
||||
{
|
||||
return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 + h * 8;
|
||||
}
|
||||
|
||||
/* No stack args of its own, so it can share a program with a tail call. */
|
||||
static __noinline __u64 call_stack(__u64 x)
|
||||
{
|
||||
return callee_stack(x + 1, x + 2, x + 3, x + 4, x + 5, x + 6, x + 7,
|
||||
x + 8);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
const volatile bool has_stack_arg = false;
|
||||
|
||||
#endif
|
||||
|
||||
/* Makes the program tail_call_reachable, though callee_stack() is not. */
|
||||
static __noinline int subprog_tail(void *ctx)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
bpf_tail_call_static(ctx, &jmp_table, 0);
|
||||
barrier_var(ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
SEC("syscall")
|
||||
int run(void *ctx)
|
||||
{
|
||||
#if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT)
|
||||
ret_stack = call_stack(base);
|
||||
#endif
|
||||
return subprog_tail(ctx);
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
|
||||
#include <vmlinux.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
|
||||
long stack_g, stack_h, stack_ret;
|
||||
|
||||
SEC("fentry/callee_stack")
|
||||
int stack_entry(unsigned long long *ctx)
|
||||
{
|
||||
stack_g = ctx[6];
|
||||
stack_h = ctx[7];
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fexit/callee_stack")
|
||||
int stack_exit(unsigned long long *ctx)
|
||||
{
|
||||
stack_ret = ctx[8];
|
||||
return 0;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
|
@ -15,4 +15,49 @@ int test_int128_arg_fexit(unsigned long long *ctx)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* The arguments come packed in the BPF convention whatever holes the
|
||||
* kernel convention left between them: an eightbyte each, and two for the
|
||||
* struct holding an __int128.
|
||||
*/
|
||||
long s_g, s_h_lo, s_h_hi, s_i, s_ret;
|
||||
|
||||
SEC("fentry/bpf_testmod_test_int128_stack")
|
||||
int test_int128_stack_fentry(unsigned long long *ctx)
|
||||
{
|
||||
s_g = ctx[6];
|
||||
s_h_lo = ctx[7];
|
||||
s_h_hi = ctx[8];
|
||||
s_i = ctx[9];
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fexit/bpf_testmod_test_int128_stack")
|
||||
int test_int128_stack_fexit(unsigned long long *ctx)
|
||||
{
|
||||
s_ret = ctx[10];
|
||||
return 0;
|
||||
}
|
||||
|
||||
long b_f_a, b_f_b, b_g, b_h, b_i_lo, b_i_hi, b_ret;
|
||||
|
||||
SEC("fentry/bpf_testmod_test_int128_backfill")
|
||||
int test_int128_backfill_fentry(unsigned long long *ctx)
|
||||
{
|
||||
b_f_a = ctx[5];
|
||||
b_f_b = ctx[6];
|
||||
b_g = ctx[7];
|
||||
b_h = ctx[8];
|
||||
b_i_lo = ctx[9];
|
||||
b_i_hi = ctx[10];
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fexit/bpf_testmod_test_int128_backfill")
|
||||
int test_int128_backfill_fexit(unsigned long long *ctx)
|
||||
{
|
||||
b_ret = ctx[11];
|
||||
return 0;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
|
|
|||
101
tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c
Normal file
101
tools/testing/selftests/bpf/progs/tracing_struct_int128_tgt.c
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
|
||||
#include <vmlinux.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
|
||||
struct int128_arg {
|
||||
__int128 v;
|
||||
};
|
||||
|
||||
__u64 base;
|
||||
__u64 ret_odd, ret_stack, ret_split, ret_unreliable;
|
||||
|
||||
#if defined(__clang__)
|
||||
|
||||
const volatile bool has_struct_arg = true;
|
||||
|
||||
/* arm64 would want v in x2 and x3, and b in x4. */
|
||||
__noinline __u64 callee_odd(__u64 a, struct int128_arg v, __u64 b)
|
||||
{
|
||||
return a + (__u64)v.v * 2 + (__u64)((unsigned __int128)v.v >> 64) * 3 +
|
||||
b * 4;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
const volatile bool has_struct_arg = false;
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(__clang__) && \
|
||||
(defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \
|
||||
defined(__BPF_FEATURE_STACK_ARGUMENT)
|
||||
|
||||
const volatile bool has_stack_arg = true;
|
||||
|
||||
/*
|
||||
* x86-64 would want h in its third and fourth stack slots, and arm64 in
|
||||
* its first and second, leaving x7 empty.
|
||||
*/
|
||||
static __noinline __u64 callee_stack(__u64 a, __u64 b, __u64 c, __u64 d,
|
||||
__u64 e, __u64 f, __u64 g,
|
||||
struct int128_arg h, __u64 i)
|
||||
{
|
||||
return a + b * 2 + c * 3 + d * 4 + e * 5 + f * 6 + g * 7 +
|
||||
(__u64)h.v * 8 + (__u64)((unsigned __int128)h.v >> 64) * 9 +
|
||||
i * 10;
|
||||
}
|
||||
|
||||
/*
|
||||
* x86-64 BPF code passes h split between r9 and its first stack slot;
|
||||
* arm64 passes it in x5 and x6, so nothing is split there.
|
||||
*/
|
||||
static __noinline __u64 callee_split(__u64 a, __u64 b, __u64 c, __u64 d,
|
||||
__u64 e, struct int128_arg h, __u64 i)
|
||||
{
|
||||
return a + b * 2 + c * 3 + d * 4 + e * 5 + (__u64)h.v * 6 +
|
||||
(__u64)((unsigned __int128)h.v >> 64) * 7 + i * 8;
|
||||
}
|
||||
|
||||
/*
|
||||
* A plain pointer makes the BTF of a static function unreliable, but f is
|
||||
* still passed on the stack. The store keeps p a pointer.
|
||||
*/
|
||||
static __noinline __u64 callee_unreliable(__u64 *p, __u64 b, __u64 c, __u64 d,
|
||||
__u64 e, __u64 f)
|
||||
{
|
||||
*p += 1;
|
||||
return *p + b * 2 + c * 3 + d * 4 + e * 5 + f * 6;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
const volatile bool has_stack_arg = false;
|
||||
|
||||
#endif
|
||||
|
||||
SEC("syscall")
|
||||
int run(void *ctx)
|
||||
{
|
||||
#if defined(__clang__)
|
||||
struct int128_arg v = { .v = ((__int128)(base + 9) << 64) | (base + 8) };
|
||||
|
||||
ret_odd = callee_odd(base + 1, v, base + 2);
|
||||
#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \
|
||||
defined(__BPF_FEATURE_STACK_ARGUMENT)
|
||||
ret_stack = callee_stack(base + 1, base + 2, base + 3, base + 4,
|
||||
base + 5, base + 6, base + 7, v, base + 10);
|
||||
ret_split = callee_split(base + 1, base + 2, base + 3, base + 4,
|
||||
base + 5, v, base + 10);
|
||||
{
|
||||
__u64 x = base;
|
||||
|
||||
ret_unreliable = callee_unreliable(&x, base + 2, base + 3,
|
||||
base + 4, base + 5, base + 6);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
|
||||
#include <vmlinux.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
#include <bpf/bpf_tracing.h>
|
||||
|
||||
struct int128_arg {
|
||||
__int128 v;
|
||||
};
|
||||
|
||||
long odd_a, odd_lo, odd_hi, odd_b, odd_ret;
|
||||
|
||||
SEC("fentry/callee_odd")
|
||||
int BPF_PROG2(odd_entry, __u64, a, struct int128_arg, v, __u64, b)
|
||||
{
|
||||
odd_a = a;
|
||||
odd_lo = (__u64)v.v;
|
||||
odd_hi = (unsigned __int128)v.v >> 64;
|
||||
odd_b = b;
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fexit/callee_odd")
|
||||
int odd_exit(unsigned long long *ctx)
|
||||
{
|
||||
odd_ret = ctx[4];
|
||||
return 0;
|
||||
}
|
||||
|
||||
long stack_g, stack_lo, stack_hi, stack_i, stack_ret;
|
||||
|
||||
SEC("fentry/callee_stack")
|
||||
int BPF_PROG2(stack_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e,
|
||||
__u64, f, __u64, g, struct int128_arg, h, __u64, i)
|
||||
{
|
||||
stack_g = g;
|
||||
stack_lo = (__u64)h.v;
|
||||
stack_hi = (unsigned __int128)h.v >> 64;
|
||||
stack_i = i;
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fexit/callee_stack")
|
||||
int stack_exit(unsigned long long *ctx)
|
||||
{
|
||||
stack_ret = ctx[10];
|
||||
return 0;
|
||||
}
|
||||
|
||||
long split_e, split_lo, split_hi, split_i, split_ret;
|
||||
|
||||
SEC("fentry/callee_split")
|
||||
int BPF_PROG2(split_entry, __u64, a, __u64, b, __u64, c, __u64, d, __u64, e,
|
||||
struct int128_arg, h, __u64, i)
|
||||
{
|
||||
split_e = e;
|
||||
split_lo = (__u64)h.v;
|
||||
split_hi = (unsigned __int128)h.v >> 64;
|
||||
split_i = i;
|
||||
return 0;
|
||||
}
|
||||
|
||||
SEC("fexit/callee_split")
|
||||
int split_exit(unsigned long long *ctx)
|
||||
{
|
||||
split_ret = ctx[8];
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Refused: the trampoline would lose the stack argument. */
|
||||
SEC("?fentry/callee_unreliable")
|
||||
int unreliable_entry(unsigned long long *ctx)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
|
|
@ -197,6 +197,43 @@ bpf_testmod_test_int128_arg(__int128 a, int b, long c)
|
|||
bpf_testmod_test_struct_arg_result = (long)a + b + c;
|
||||
return bpf_testmod_test_struct_arg_result;
|
||||
}
|
||||
|
||||
struct bpf_testmod_struct_int128 {
|
||||
__int128 a;
|
||||
};
|
||||
|
||||
/*
|
||||
* x86-64 puts g in the first stack slot and h, aligned to 16 bytes, in
|
||||
* the third and fourth, leaving a hole. arm64 leaves x7 empty and puts h
|
||||
* in the first two stack slots. Either way h and i are not where the BPF
|
||||
* convention packs them, right after g.
|
||||
*/
|
||||
noinline long
|
||||
bpf_testmod_test_int128_stack(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f,
|
||||
u64 g, struct bpf_testmod_struct_int128 h, u64 i)
|
||||
{
|
||||
bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 +
|
||||
f * 6 + g * 7 + (u64)h.a * 8 +
|
||||
(u64)((unsigned __int128)h.a >> 64) * 9 + i * 10;
|
||||
return bpf_testmod_test_struct_arg_result;
|
||||
}
|
||||
|
||||
/*
|
||||
* x86-64 has no register left for f after e, so f goes to the stack and g
|
||||
* takes R9 back; h goes to the third stack slot and i, aligned to 16
|
||||
* bytes, to the fifth and sixth. arm64 has registers up to g, puts h in
|
||||
* the first stack slot and i in the third and fourth.
|
||||
*/
|
||||
noinline long
|
||||
bpf_testmod_test_int128_backfill(u64 a, u64 b, u64 c, u64 d, u64 e,
|
||||
struct bpf_testmod_struct_arg_2 f, u64 g, u64 h,
|
||||
struct bpf_testmod_struct_int128 i)
|
||||
{
|
||||
bpf_testmod_test_struct_arg_result = a + b * 2 + c * 3 + d * 4 + e * 5 +
|
||||
f.a * 6 + f.b * 7 + g * 8 + h * 9 + (u64)i.a * 10 +
|
||||
(u64)((unsigned __int128)i.a >> 64) * 11;
|
||||
return bpf_testmod_test_struct_arg_result;
|
||||
}
|
||||
#endif
|
||||
|
||||
__weak noinline void bpf_testmod_looooooooooooooooooooooooooooooong_name(void)
|
||||
|
|
@ -683,6 +720,12 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj,
|
|||
#ifdef __SIZEOF_INT128__
|
||||
(void)bpf_testmod_test_int128_ret(i);
|
||||
(void)bpf_testmod_test_int128_arg((__int128)1, 2, 3);
|
||||
(void)bpf_testmod_test_int128_stack(1, 2, 3, 4, 5, 6, 7,
|
||||
(struct bpf_testmod_struct_int128){ .a = ((__int128)9 << 64) | 8 },
|
||||
10);
|
||||
(void)bpf_testmod_test_int128_backfill(1, 2, 3, 4, 5,
|
||||
(struct bpf_testmod_struct_arg_2){ .a = 6, .b = 7 }, 8, 9,
|
||||
(struct bpf_testmod_struct_int128){ .a = ((__int128)11 << 64) | 10 });
|
||||
#endif
|
||||
|
||||
(void)trace_bpf_testmod_test_raw_tp_null_tp(NULL);
|
||||
|
|
@ -1047,6 +1090,29 @@ __bpf_kfunc u64 bpf_kfunc_call_test_i128_arg(u64 a, u64 b, __int128 v)
|
|||
return a + b * 2 + (u64)v * 3 + (u64)((unsigned __int128)v >> 64) * 4;
|
||||
}
|
||||
|
||||
__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd(u64 a, __int128 v, u64 b)
|
||||
{
|
||||
return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 + b * 4;
|
||||
}
|
||||
|
||||
__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_odd_many(u64 a, __int128 v, u64 b, u64 c,
|
||||
u64 d, u64 e, u64 f, __int128 w)
|
||||
{
|
||||
return a + (u64)v * 2 + (u64)((unsigned __int128)v >> 64) * 3 +
|
||||
b * 4 + c * 5 + d * 6 + e * 7 + f * 8 +
|
||||
(u64)w * 9 + (u64)((unsigned __int128)w >> 64) * 10;
|
||||
}
|
||||
|
||||
__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_stack_hole(u64 a, u64 b, u64 c, u64 d, u64 e,
|
||||
struct prog_test_pair_arg s,
|
||||
u64 f, u64 g,
|
||||
struct prog_test_i128_arg w)
|
||||
{
|
||||
return a + b * 2 + c * 3 + d * 4 + e * 5 + s.lo * 6 + s.hi * 7 +
|
||||
f * 8 + g * 9 + (u64)w.v * 10 +
|
||||
(u64)((unsigned __int128)w.v >> 64) * 11;
|
||||
}
|
||||
|
||||
__bpf_kfunc u64 bpf_kfunc_call_test_i128_arg_pad(u64 a, u64 b, u64 c, u64 d, u64 e,
|
||||
u64 f, u64 g, __int128 v)
|
||||
{
|
||||
|
|
@ -1770,6 +1836,9 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep)
|
|||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii)
|
||||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg)
|
||||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg)
|
||||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd)
|
||||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_odd_many)
|
||||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_stack_hole)
|
||||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128_arg_pad)
|
||||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_nofit)
|
||||
BTF_ID_FLAGS(func, bpf_kfunc_call_test_pair_arg_tail)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,12 @@ struct prog_test_pair_arg { /* 16 bytes: two argument registers */
|
|||
__u64 hi;
|
||||
};
|
||||
|
||||
#ifdef __SIZEOF_INT128__
|
||||
struct prog_test_i128_arg { /* 16 bytes, aligned to 16 like its member */
|
||||
__int128 v;
|
||||
};
|
||||
#endif
|
||||
|
||||
struct prog_test_ptr_arg { /* 16 bytes, but holds a pointer */
|
||||
void *p;
|
||||
__u64 x;
|
||||
|
|
@ -234,6 +240,13 @@ struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym;
|
|||
__u64 bpf_kfunc_call_test_pair_arg(__u64 a, struct prog_test_pair_arg s, __u64 b) __ksym;
|
||||
#ifdef __SIZEOF_INT128__
|
||||
__u64 bpf_kfunc_call_test_i128_arg(__u64 a, __u64 b, __int128 v) __ksym;
|
||||
__u64 bpf_kfunc_call_test_i128_arg_odd(__u64 a, __int128 v, __u64 b) __ksym;
|
||||
__u64 bpf_kfunc_call_test_i128_arg_odd_many(__u64 a, __int128 v, __u64 b, __u64 c,
|
||||
__u64 d, __u64 e, __u64 f, __int128 w) __ksym;
|
||||
__u64 bpf_kfunc_call_test_i128_arg_stack_hole(__u64 a, __u64 b, __u64 c, __u64 d,
|
||||
__u64 e, struct prog_test_pair_arg s,
|
||||
__u64 f, __u64 g,
|
||||
struct prog_test_i128_arg w) __ksym;
|
||||
__u64 bpf_kfunc_call_test_i128_arg_pad(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e,
|
||||
__u64 f, __u64 g, __int128 v) __ksym;
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user