github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 04:34:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf, x86: Convert struct_ops arena arguments in the trampoline

Browse Source 
Implement the struct_ops arena argument conversion on x86. save_args()
receives the arena base from bpf_tramp_arena_base() and consults the
btf_func_model argument flags as it copies each native argument into the
BPF ctx, routing a marked argument through RAX:

  movl %esrc, %eax      /* truncate and clear the upper 32 bits */
  subl $base_lo, %eax
  movq %rax, ctx_slot

A nullable argument tests the full 64-bit kernel pointer first:

  movq  %rsrc, %rax
  testq %rax, %rax
  jz    1f
  subl  $base_lo, %eax
1:
  movq  %rax, ctx_slot

The 32-bit subtraction is sufficient since (u32)(kaddr - base) ==
(u32)kaddr - (u32)base, and it clears the upper half as the JITs require
of arena pointer registers. Stack-passed arguments already reload
through RAX, so only the subtraction (and the NULL test) is inserted
there.

Keep arena and nullable classification in btf_func_model.
bpf_tramp_arena_base() returns a base only for a single-program
struct_ops indirect trampoline; other trampolines pass zero and perform
no conversion. The size probe reruns the same emission with the same
model and nodes, so the image size matches by construction.

With both the kfunc and struct_ops directions implemented, flip
bpf_jit_supports_arena_args() on for x86.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://patch.msgid.link/20260808003938.3486067-8-memxor@gmail.com
Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
This commit is contained in:
Tejun Heo 2026-08-08 02:39:27 +02:00 committed by Eduard Zingerman
parent 41b7552230
commit 5129e9be21
1 changed files with 51 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
arch/x86/net/bpf_jit_comp.c
View File

@ -3048,8 +3048,35 @@ static int get_nr_used_regs(const struct btf_func_model *m)
return nr_used_regs;
}
/*
* Convert an arena kernel address into the arena pointer form on its way
* into the BPF ctx, rax = (u32)(src - kern_vm_start). A nullable arg
* preserves NULL, tested on the full 64-bit kernel pointer. The 32-bit
* subtraction both truncates and clears the upper half, so the stored
* value satisfies the JIT invariant for arena pointer registers.
*/
static void emit_arena_arg_conv(u8 **pprog, u32 src_reg, bool nullable, u32 base_lo)
{
u8 *prog = *pprog;
if (nullable) {
if (src_reg != BPF_REG_0)
emit_mov_reg(&prog, true, BPF_REG_0, src_reg);
/* test rax, rax; jz over the 5-byte sub */
EMIT3(0x48, 0x85, 0xC0);
EMIT2(X86_JE, 5);
} else if (src_reg != BPF_REG_0) {
emit_mov_reg(&prog, false, BPF_REG_0, src_reg);
}
/* sub eax, base_lo */
EMIT1_off32(0x2D, base_lo);
*pprog = prog;
}
static void save_args(const struct btf_func_model *m, u8 **prog,
int stack_size, bool for_call_origin, u32 flags)
int stack_size, bool for_call_origin, u32 flags,
u64 arena_base)
{
int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0;
bool use_jmp = bpf_trampoline_use_jmp(flags);
@ -3061,6 +3088,9 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
* mov QWORD PTR [rbp-0x8],rsi
*/
for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) {
bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG);
bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG;
arg_regs = (m->arg_size[i] + 7) / 8;
/* According to the research of Yonghong, struct members
@ -3094,6 +3124,9 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
for (j = 0; j < arg_regs; j++) {
emit_ldx(prog, BPF_DW, BPF_REG_0, BPF_REG_FP,
nr_stack_slots * 8 + 16 + (!use_jmp) * 8);
if (arena_arg)
emit_arena_arg_conv(prog, BPF_REG_0, nullable,
(u32)arena_base);
emit_stx(prog, BPF_DW, BPF_REG_FP, BPF_REG_0,
-stack_size);
@ -3114,9 +3147,13 @@ static void save_args(const struct btf_func_model *m, u8 **prog,
/* copy the arguments from regs into stack */
for (j = 0; j < arg_regs; j++) {
emit_stx(prog, BPF_DW, BPF_REG_FP,
nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs,
-stack_size);
u32 src = nr_regs == 5 ? X86_REG_R9 : BPF_REG_1 + nr_regs;
if (arena_arg) {
emit_arena_arg_conv(prog, src, nullable, (u32)arena_base);
src = BPF_REG_0;
}
emit_stx(prog, BPF_DW, BPF_REG_FP, src, -stack_size);
stack_size -= 8;
nr_regs++;
}
@ -3412,6 +3449,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
void *orig_call = func_addr;
int cookie_off, cookie_cnt;
u8 **branches = NULL;
u64 arena_base;
u64 func_meta;
u8 *prog;
bool save_ret;
@ -3424,6 +3462,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) &&
(flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET)));
arena_base = bpf_tramp_arena_base(m, tnodes, flags);
for (i = 0; i < m->nr_args; i++)
nr_regs += (m->arg_size[i] + 7) / 8 - 1;
@ -3558,7 +3598,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
emit_store_stack_imm64(&prog, BPF_REG_0, -ip_off, (long)func_addr);
}
save_args(m, &prog, regs_off, false, flags);
save_args(m, &prog, regs_off, false, flags, arena_base);
if (flags & BPF_TRAMP_F_CALL_ORIG) {
/* arg1: mov rdi, im */
@ -3600,7 +3640,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
if (flags & BPF_TRAMP_F_CALL_ORIG) {
restore_regs(m, &prog, regs_off);
save_args(m, &prog, arg_stack_off, true, flags);
save_args(m, &prog, arg_stack_off, true, flags, 0);
if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) {
/* Before calling the original function, load the
@ -4101,6 +4141,11 @@ bool bpf_jit_supports_stack_args(void)
return true;
}
bool bpf_jit_supports_arena_args(void)
{
return true;
}
void *bpf_arch_text_copy(void *dst, void *src, size_t len)
{
if (text_poke_copy(dst, src, len) == NULL)