github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-23 05:04:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf, x86: JIT __arena kfunc argument rebasing

Browse Source 
Implement arena argument rebasing for kfunc calls on x86. R12 already
holds kern_vm_start whenever the prog has an arena, so each tagged
argument costs two instructions emitted right before the call:

  movl %eN, %eN         /* truncate, clear the upper 32 bits */
  addq %r12, %rN

A nullable argument tests the truncated value and jumps over the add:

  movl  %eN, %eN
  testl %eN, %eN
  jz    1f
  addq  %r12, %rN
1:

addq carries a REX prefix for every argument register and is always
three bytes, so the jz displacement is constant. The sequence is native
code generated after constant blinding has run on the BPF instruction
stream, so blinding never sees the rebase and needs no special handling.

bpf_jit_supports_arena_args() is not flipped yet; that happens when the
struct_ops trampoline side is in place as well.

Signed-off-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://patch.msgid.link/20260808003938.3486067-7-memxor@gmail.com
Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
This commit is contained in:
Tejun Heo 2026-08-08 02:39:26 +02:00 committed by Eduard Zingerman
parent f6c33c4479
commit 41b7552230
1 changed files with 50 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
arch/x86/net/bpf_jit_comp.c
View File

@ -1678,6 +1678,50 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip,
return 0;
}
/*
* Rebase the __arena args of a kfunc call to arena kernel addresses,
* rN = kern_vm_start + (u32)rN, with R12 holding kern_vm_start. A nullable
* arg preserves NULL by skipping the add, tested on the truncated value as
* arena NULL is offset 0. Return the number of emitted bytes.
*/
static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog,
const struct bpf_insn *insn, u8 **pprog)
{
const struct btf_func_model *fm;
u8 *prog = *pprog;
u8 *start = prog;
int i;
fm = bpf_jit_find_kfunc_model(bpf_prog, insn);
if (!fm)
return -EINVAL;
for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) {
u8 flags = fm->arg_flags[i];
u32 reg = BPF_REG_1 + i;
if (!(flags & BTF_FMODEL_ARENA_ARG))
continue;
if (WARN_ON_ONCE(!bpf_prog->aux->arena))
return -EINVAL;
/* mov eN, eN: truncate and clear the upper 32 bits */
emit_mov_reg(&prog, false, reg, reg);
if (flags & BTF_FMODEL_NULLABLE_ARG) {
/* test eN, eN; jz over the 3-byte add */
maybe_emit_mod(&prog, reg, reg, false);
EMIT2(0x85, add_2reg(0xC0, reg, reg));
EMIT2(X86_JE, 3);
}
/* add rN, r12 */
maybe_emit_mod(&prog, reg, X86_REG_R12, true);
EMIT2(0x01, add_2reg(0xC0, reg, X86_REG_R12));
}
*pprog = prog;
return prog - start;
}
static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image,
u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding)
{
@ -2588,6 +2632,12 @@ st: insn_off = insn->off;
}
if (!imm32)
return -EINVAL;
if (src_reg == BPF_PSEUDO_KFUNC_CALL) {
err = emit_kfunc_arena_args(bpf_prog, insn, &prog);
if (err < 0)
return err;
ip += err;
}
if (priv_frame_ptr) {
push_r9(&prog);
ip += 2;