github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: Pass a pointer to arch_bpf_timed_may_goto()

Browse Source 
arch_bpf_timed_may_goto() gets the offset of count and timestamp in the
stack frame in BPF_REG_AX. Arch code adds the frame pointer to it, and
calls bpf_check_timed_may_goto() with the result. On arm64, riscv,
loongarch, powerpc and s390 the asm function does it. On x86-64 the JIT
does it, because the frame pointer is in another register when the prog has
a private stack.

The next patch moves count and timestamp out of the stack frame. So pass
the pointer in BPF_REG_AX, and let the verifier emit the addition:

  before                           after

  r12 = -16                        r12 = r10
  call arch_bpf_timed_may_goto     r12 += -16
                                   call arch_bpf_timed_may_goto

The JITs know which register r10 is. The asm functions have a move in place
of the addition, and the special case in the x86-64 JIT goes away.

These insns run when the count is used up: in each 65535th iteration. The
cost of an iteration does not change.

Three tests have the insns of may_goto in them. Update them.

Tested on x86-64, and on arm64 and s390 in QEMU. riscv, loongarch and
powerpc are compile tested only.

Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://patch.msgid.link/20261031215543.682284-7-alexei.starovoitov@gmail.com
This commit is contained in:
Alexei Starovoitov 2026-10-31 21:55:40 +00:00
parent 85b6a11252
commit 78821f4d8d
No known key found for this signature in database
11 changed files with 51 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
arch/arm64/net/bpf_timed_may_goto.S
View File

@ -17,16 +17,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
str x7, [sp, #80]
/*
* Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP
* (x25) to get the pointer to count and timestamp and pass it as the
* first argument in x0.
*
* Before generating the call to arch_bpf_timed_may_goto, the verifier
* generates a load instruction using FP, i.e. REG_AX = *(u64 *)(FP -
* stack_off_cnt), so BPF_REG_FP (x25) is always set up by the arm64
* jit in this case.
* The pointer to count and timestamp was passed in BPF_REG_AX (x9),
* pass it as the first argument in x0.
*/
add x0, x9, x25
mov x0, x9
bl bpf_check_timed_may_goto
/* BPF_REG_AX(x9) will be stored into count, so move return value to it. */
mov x9, x0

10
arch/loongarch/net/bpf_timed_may_goto.S
View File

@ -22,14 +22,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
st.d a4, sp, 48
/*
* BPF_REG_AX (t0) holds the offset passed in by the verifier;
* add it to BPF_REG_FP (s4) to get the pointer to the count and
* timestamp, then pass it as the first argument in a0.
*
* The verifier emits a load using FP right before this call,
* so BPF_REG_FP (s4) is always set up by the JIT in this case.
* BPF_REG_AX (t0) holds the pointer to the count and timestamp;
* pass it as the first argument in a0.
*/
add.d a0, t0, s4
move a0, t0
bl bpf_check_timed_may_goto
/* BPF_REG_AX (t0) will be stored into count, so move the return value to it. */
move t0, a0

9
arch/powerpc/net/bpf_timed_may_goto.S
View File

@ -8,7 +8,7 @@
* arch_bpf_timed_may_goto() trampoline for powerpc64
*
* Custom BPF convention (verifier/JIT):
* - input: stack offset in BPF_REG_AX (r12)
* - input: pointer to count and timestamp in BPF_REG_AX (r12)
* - output: updated count in BPF_REG_AX (r12)
*
* Call bpf_check_timed_may_goto(ptr) with normal powerpc64 ABI:
@ -31,11 +31,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
std r7, 64(r1)
std r8, 72(r1)
/*
* r3 = BPF_REG_FP + BPF_REG_AX
* BPF_REG_FP is r31; BPF_REG_AX is r12 (stack offset in bytes).
*/
add r3, r31, r12
/* r3 = BPF_REG_AX (r12) */
mr r3, r12
bl CFUNC(bpf_check_timed_may_goto)
/* Put return value back into AX */

10
arch/riscv/net/bpf_timed_may_goto.S
View File

@ -6,13 +6,11 @@
/*
* Trampoline for the BPF timed may_goto loop bound. Custom calling convention:
* - input: stack offset in BPF_REG_AX (t0)
* - input: pointer to count and timestamp in BPF_REG_AX (t0)
* - output: updated count in BPF_REG_AX (t0)
*
* Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where
* ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) and stack
* arguments 6-8 (a5-a7) are saved across the call; BPF_REG_FP (s5) is
* callee-saved and needs no saving.
* Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI. BPF R0-R5
* (t6, a0-a4) and stack arguments 6-8 (a5-a7) are saved across the call.
*/
SYM_FUNC_START(arch_bpf_timed_may_goto)
@ -33,7 +31,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
REG_S a6, 7*SZREG(sp)
REG_S a7, 8*SZREG(sp)
add a0, t0, s5
mv a0, t0
call bpf_check_timed_may_goto
mv t0, a0

6
arch/s390/net/bpf_jit_comp.c
View File

@ -1877,9 +1877,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
if ((void *)func == arch_bpf_timed_may_goto) {
/*
* arch_bpf_timed_may_goto() has a special ABI: the
* parameters are in BPF_REG_AX and BPF_REG_10; the
* return value is in BPF_REG_AX; and all GPRs except
* REG_W0, REG_W1, and BPF_REG_AX are callee-saved.
* parameter is in BPF_REG_AX; the return value is in
* BPF_REG_AX; and all GPRs except REG_W0, REG_W1, and
* BPF_REG_AX are callee-saved.
*/
/* brasl %r0,func */

8
arch/s390/net/bpf_timed_may_goto.S
View File

@ -21,9 +21,9 @@
SYM_FUNC_START(arch_bpf_timed_may_goto)
/*
* This function has a special ABI: the parameters are in %r12 and
* %r13; the return value is in %r12; all GPRs except %r0, %r1, and
* %r12 are callee-saved; and the return address is in %r0.
* This function has a special ABI: the parameter is in %r12; the
* return value is in %r12; all GPRs except %r0, %r1, and %r12 are
* callee-saved; and the return address is in %r0.
*/
stmg %r2,%r5,FRAME_OFF+R2_OFF(%r15)
stg %r14,FRAME_OFF+R14_OFF(%r15)
@ -33,7 +33,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
lay %r15,-FRAME_SIZE(%r15)
stg %r1,__SF_BACKCHAIN(%r15)
lay %r2,0(%r12,%r13)
lgr %r2,%r12
brasl %r14,bpf_check_timed_may_goto
lgr %r12,%r2

12
arch/x86/net/bpf_jit_comp.c
View File

@ -3441,18 +3441,6 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
ip += emit_kfunc_arg_moves(fm, outgoing_arg_base -
outgoing_rsp, &prog);
}
if (func == (u8 *)arch_bpf_timed_may_goto) {
u32 fp = priv_frame_ptr ? X86_REG_R9 : BPF_REG_FP;
/*
* AX has the offset of count and timestamp
* in the stack. Turn it into a pointer.
* add r10, rbp or add r10, r9
*/
maybe_emit_mod(&prog, BPF_REG_AX, fp, true);
EMIT2(0x01, add_2reg(0xC0, BPF_REG_AX, fp));
ip += 3;
}
if (priv_frame_ptr) {
push_r9(&prog);
ip += 2;

5
arch/x86/net/bpf_timed_may_goto.S
View File

@ -25,10 +25,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
/* Save stack argument 6. */
pushq %r9
/*
* r10 has the pointer to count and timestamp. JIT added the frame
* pointer to the offset. Pass it as first argument.
*/
/* r10 has the pointer to count and timestamp. Pass it as first argument. */
movq %r10, %rdi
/* Emit call depth accounting for call below. */

9
kernel/bpf/fixups.c
View File

@ -2083,14 +2083,15 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
if (bpf_is_may_goto_insn(insn) && timed_may_goto) {
int stack_off_cnt = -stack_depth - 16;
/*
* AX is used as an argument to pass in stack_off_cnt
* (to add to r10/fp), and also as the return value of
* AX is used as an argument to pass in the pointer to
* count and timestamp, and also as the return value of
* the call to arch_bpf_timed_may_goto.
*/
struct bpf_insn tail[] = {
BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1),
BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2),
BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt),
BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 3),
BPF_MOV64_REG(BPF_REG_AX, BPF_REG_10),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, stack_off_cnt),
BPF_EMIT_CALL(arch_bpf_timed_may_goto),
BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt),
};

30
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
View File

@ -914,15 +914,16 @@ __xlated("4: r0 = &(void __percpu *)(r0)")
__xlated("...")
/* may_goto expansion starts */
__xlated("6: r12 = *(u64 *)(r10 -24)")
__xlated("7: if r12 == 0x0 goto pc+6")
__xlated("7: if r12 == 0x0 goto pc+7")
__xlated("8: r12 -= 1")
__xlated("9: if r12 != 0x0 goto pc+2")
__xlated("10: r12 = -24")
__xlated("11: call unknown")
__xlated("12: *(u64 *)(r10 -24) = r12")
__xlated("9: if r12 != 0x0 goto pc+3")
__xlated("10: r12 = r10")
__xlated("11: r12 += -24")
__xlated("12: call unknown")
__xlated("13: *(u64 *)(r10 -24) = r12")
/* may_goto expansion ends */
__xlated("13: *(u64 *)(r10 -8) = r1")
__xlated("14: exit")
__xlated("14: *(u64 *)(r10 -8) = r1")
__xlated("15: exit")
__success
__naked void may_goto_interaction_x86_64(void)
{
@ -955,15 +956,16 @@ __xlated("2: r1 = 1")
__xlated("3: call bpf_get_smp_processor_id")
/* may_goto expansion starts */
__xlated("4: r12 = *(u64 *)(r10 -24)")
__xlated("5: if r12 == 0x0 goto pc+6")
__xlated("5: if r12 == 0x0 goto pc+7")
__xlated("6: r12 -= 1")
__xlated("7: if r12 != 0x0 goto pc+2")
__xlated("8: r12 = -24")
__xlated("9: call unknown")
__xlated("10: *(u64 *)(r10 -24) = r12")
__xlated("7: if r12 != 0x0 goto pc+3")
__xlated("8: r12 = r10")
__xlated("9: r12 += -24")
__xlated("10: call unknown")
__xlated("11: *(u64 *)(r10 -24) = r12")
/* may_goto expansion ends */
__xlated("11: *(u64 *)(r10 -8) = r1")
__xlated("12: exit")
__xlated("12: *(u64 *)(r10 -8) = r1")
__xlated("13: exit")
__success
__naked void may_goto_interaction(void)
{

17
tools/testing/selftests/bpf/progs/verifier_may_goto_1.c
View File

@ -91,15 +91,16 @@ __arch_loongarch
__xlated("0: *(u64 *)(r10 -16) = 65535")
__xlated("1: *(u64 *)(r10 -8) = 0")
__xlated("2: r12 = *(u64 *)(r10 -16)")
__xlated("3: if r12 == 0x0 goto pc+6")
__xlated("3: if r12 == 0x0 goto pc+7")
__xlated("4: r12 -= 1")
__xlated("5: if r12 != 0x0 goto pc+2")
__xlated("6: r12 = -16")
__xlated("7: call unknown")
__xlated("8: *(u64 *)(r10 -16) = r12")
__xlated("9: r0 = 1")
__xlated("10: r0 = 2")
__xlated("11: exit")
__xlated("5: if r12 != 0x0 goto pc+3")
__xlated("6: r12 = r10")
__xlated("7: r12 += -16")
__xlated("8: call unknown")
__xlated("9: *(u64 *)(r10 -16) = r12")
__xlated("10: r0 = 1")
__xlated("11: r0 = 2")
__xlated("12: exit")
__success
__naked void may_goto_batch_2(void)
{