From 78821f4d8d1836c6cb9a21a780092906fc82ee09 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 31 Oct 2026 21:55:40 +0000 Subject: [PATCH] bpf: Pass a pointer to arch_bpf_timed_may_goto() arch_bpf_timed_may_goto() gets the offset of count and timestamp in the stack frame in BPF_REG_AX. Arch code adds the frame pointer to it, and calls bpf_check_timed_may_goto() with the result. On arm64, riscv, loongarch, powerpc and s390 the asm function does it. On x86-64 the JIT does it, because the frame pointer is in another register when the prog has a private stack. The next patch moves count and timestamp out of the stack frame. So pass the pointer in BPF_REG_AX, and let the verifier emit the addition: before after r12 = -16 r12 = r10 call arch_bpf_timed_may_goto r12 += -16 call arch_bpf_timed_may_goto The JITs know which register r10 is. The asm functions have a move in place of the addition, and the special case in the x86-64 JIT goes away. These insns run when the count is used up: in each 65535th iteration. The cost of an iteration does not change. Three tests have the insns of may_goto in them. Update them. Tested on x86-64, and on arm64 and s390 in QEMU. riscv, loongarch and powerpc are compile tested only. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261031215543.682284-7-alexei.starovoitov@gmail.com --- arch/arm64/net/bpf_timed_may_goto.S | 12 ++------ arch/loongarch/net/bpf_timed_may_goto.S | 10 ++----- arch/powerpc/net/bpf_timed_may_goto.S | 9 ++---- arch/riscv/net/bpf_timed_may_goto.S | 10 +++---- arch/s390/net/bpf_jit_comp.c | 6 ++-- arch/s390/net/bpf_timed_may_goto.S | 8 ++--- arch/x86/net/bpf_jit_comp.c | 12 -------- arch/x86/net/bpf_timed_may_goto.S | 5 +--- kernel/bpf/fixups.c | 9 +++--- .../bpf/progs/verifier_bpf_fastcall.c | 30 ++++++++++--------- .../selftests/bpf/progs/verifier_may_goto_1.c | 17 ++++++----- 11 files changed, 51 insertions(+), 77 deletions(-) diff --git a/arch/arm64/net/bpf_timed_may_goto.S b/arch/arm64/net/bpf_timed_may_goto.S index 1b4a4f91ffc9..f8732b13bb1e 100644 --- a/arch/arm64/net/bpf_timed_may_goto.S +++ b/arch/arm64/net/bpf_timed_may_goto.S @@ -17,16 +17,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) str x7, [sp, #80] /* - * Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP - * (x25) to get the pointer to count and timestamp and pass it as the - * first argument in x0. - * - * Before generating the call to arch_bpf_timed_may_goto, the verifier - * generates a load instruction using FP, i.e. REG_AX = *(u64 *)(FP - - * stack_off_cnt), so BPF_REG_FP (x25) is always set up by the arm64 - * jit in this case. + * The pointer to count and timestamp was passed in BPF_REG_AX (x9), + * pass it as the first argument in x0. */ - add x0, x9, x25 + mov x0, x9 bl bpf_check_timed_may_goto /* BPF_REG_AX(x9) will be stored into count, so move return value to it. */ mov x9, x0 diff --git a/arch/loongarch/net/bpf_timed_may_goto.S b/arch/loongarch/net/bpf_timed_may_goto.S index fa128acdb108..4aaca029404b 100644 --- a/arch/loongarch/net/bpf_timed_may_goto.S +++ b/arch/loongarch/net/bpf_timed_may_goto.S @@ -22,14 +22,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) st.d a4, sp, 48 /* - * BPF_REG_AX (t0) holds the offset passed in by the verifier; - * add it to BPF_REG_FP (s4) to get the pointer to the count and - * timestamp, then pass it as the first argument in a0. - * - * The verifier emits a load using FP right before this call, - * so BPF_REG_FP (s4) is always set up by the JIT in this case. + * BPF_REG_AX (t0) holds the pointer to the count and timestamp; + * pass it as the first argument in a0. */ - add.d a0, t0, s4 + move a0, t0 bl bpf_check_timed_may_goto /* BPF_REG_AX (t0) will be stored into count, so move the return value to it. */ move t0, a0 diff --git a/arch/powerpc/net/bpf_timed_may_goto.S b/arch/powerpc/net/bpf_timed_may_goto.S index 84ecf6fa7f5d..b8ae9885eb31 100644 --- a/arch/powerpc/net/bpf_timed_may_goto.S +++ b/arch/powerpc/net/bpf_timed_may_goto.S @@ -8,7 +8,7 @@ * arch_bpf_timed_may_goto() trampoline for powerpc64 * * Custom BPF convention (verifier/JIT): - * - input: stack offset in BPF_REG_AX (r12) + * - input: pointer to count and timestamp in BPF_REG_AX (r12) * - output: updated count in BPF_REG_AX (r12) * * Call bpf_check_timed_may_goto(ptr) with normal powerpc64 ABI: @@ -31,11 +31,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) std r7, 64(r1) std r8, 72(r1) - /* - * r3 = BPF_REG_FP + BPF_REG_AX - * BPF_REG_FP is r31; BPF_REG_AX is r12 (stack offset in bytes). - */ - add r3, r31, r12 + /* r3 = BPF_REG_AX (r12) */ + mr r3, r12 bl CFUNC(bpf_check_timed_may_goto) /* Put return value back into AX */ diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index 6ec37f634aff..2ce1c89c913e 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -6,13 +6,11 @@ /* * Trampoline for the BPF timed may_goto loop bound. Custom calling convention: - * - input: stack offset in BPF_REG_AX (t0) + * - input: pointer to count and timestamp in BPF_REG_AX (t0) * - output: updated count in BPF_REG_AX (t0) * - * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) and stack - * arguments 6-8 (a5-a7) are saved across the call; BPF_REG_FP (s5) is - * callee-saved and needs no saving. + * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI. BPF R0-R5 + * (t6, a0-a4) and stack arguments 6-8 (a5-a7) are saved across the call. */ SYM_FUNC_START(arch_bpf_timed_may_goto) @@ -33,7 +31,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S a6, 7*SZREG(sp) REG_S a7, 8*SZREG(sp) - add a0, t0, s5 + mv a0, t0 call bpf_check_timed_may_goto mv t0, a0 diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index b6224da9f6e8..3ab880fc17a6 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1877,9 +1877,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if ((void *)func == arch_bpf_timed_may_goto) { /* * arch_bpf_timed_may_goto() has a special ABI: the - * parameters are in BPF_REG_AX and BPF_REG_10; the - * return value is in BPF_REG_AX; and all GPRs except - * REG_W0, REG_W1, and BPF_REG_AX are callee-saved. + * parameter is in BPF_REG_AX; the return value is in + * BPF_REG_AX; and all GPRs except REG_W0, REG_W1, and + * BPF_REG_AX are callee-saved. */ /* brasl %r0,func */ diff --git a/arch/s390/net/bpf_timed_may_goto.S b/arch/s390/net/bpf_timed_may_goto.S index 06f567a460d7..05f532f8c639 100644 --- a/arch/s390/net/bpf_timed_may_goto.S +++ b/arch/s390/net/bpf_timed_may_goto.S @@ -21,9 +21,9 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* - * This function has a special ABI: the parameters are in %r12 and - * %r13; the return value is in %r12; all GPRs except %r0, %r1, and - * %r12 are callee-saved; and the return address is in %r0. + * This function has a special ABI: the parameter is in %r12; the + * return value is in %r12; all GPRs except %r0, %r1, and %r12 are + * callee-saved; and the return address is in %r0. */ stmg %r2,%r5,FRAME_OFF+R2_OFF(%r15) stg %r14,FRAME_OFF+R14_OFF(%r15) @@ -33,7 +33,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) lay %r15,-FRAME_SIZE(%r15) stg %r1,__SF_BACKCHAIN(%r15) - lay %r2,0(%r12,%r13) + lgr %r2,%r12 brasl %r14,bpf_check_timed_may_goto lgr %r12,%r2 diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c9419bc65f08..4c3e09d98131 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3441,18 +3441,6 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * ip += emit_kfunc_arg_moves(fm, outgoing_arg_base - outgoing_rsp, &prog); } - if (func == (u8 *)arch_bpf_timed_may_goto) { - u32 fp = priv_frame_ptr ? X86_REG_R9 : BPF_REG_FP; - - /* - * AX has the offset of count and timestamp - * in the stack. Turn it into a pointer. - * add r10, rbp or add r10, r9 - */ - maybe_emit_mod(&prog, BPF_REG_AX, fp, true); - EMIT2(0x01, add_2reg(0xC0, BPF_REG_AX, fp)); - ip += 3; - } if (priv_frame_ptr) { push_r9(&prog); ip += 2; diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index dd2dde89836b..da42de7cca98 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -25,10 +25,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* Save stack argument 6. */ pushq %r9 - /* - * r10 has the pointer to count and timestamp. JIT added the frame - * pointer to the offset. Pass it as first argument. - */ + /* r10 has the pointer to count and timestamp. Pass it as first argument. */ movq %r10, %rdi /* Emit call depth accounting for call below. */ diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index bb6b57f5e144..5cbab55d84be 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2083,14 +2083,15 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (bpf_is_may_goto_insn(insn) && timed_may_goto) { int stack_off_cnt = -stack_depth - 16; /* - * AX is used as an argument to pass in stack_off_cnt - * (to add to r10/fp), and also as the return value of + * AX is used as an argument to pass in the pointer to + * count and timestamp, and also as the return value of * the call to arch_bpf_timed_may_goto. */ struct bpf_insn tail[] = { BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), - BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), - BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), + BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 3), + BPF_MOV64_REG(BPF_REG_AX, BPF_REG_10), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, stack_off_cnt), BPF_EMIT_CALL(arch_bpf_timed_may_goto), BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), }; diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index e0c389102db9..faedf36eef2a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -914,15 +914,16 @@ __xlated("4: r0 = &(void __percpu *)(r0)") __xlated("...") /* may_goto expansion starts */ __xlated("6: r12 = *(u64 *)(r10 -24)") -__xlated("7: if r12 == 0x0 goto pc+6") +__xlated("7: if r12 == 0x0 goto pc+7") __xlated("8: r12 -= 1") -__xlated("9: if r12 != 0x0 goto pc+2") -__xlated("10: r12 = -24") -__xlated("11: call unknown") -__xlated("12: *(u64 *)(r10 -24) = r12") +__xlated("9: if r12 != 0x0 goto pc+3") +__xlated("10: r12 = r10") +__xlated("11: r12 += -24") +__xlated("12: call unknown") +__xlated("13: *(u64 *)(r10 -24) = r12") /* may_goto expansion ends */ -__xlated("13: *(u64 *)(r10 -8) = r1") -__xlated("14: exit") +__xlated("14: *(u64 *)(r10 -8) = r1") +__xlated("15: exit") __success __naked void may_goto_interaction_x86_64(void) { @@ -955,15 +956,16 @@ __xlated("2: r1 = 1") __xlated("3: call bpf_get_smp_processor_id") /* may_goto expansion starts */ __xlated("4: r12 = *(u64 *)(r10 -24)") -__xlated("5: if r12 == 0x0 goto pc+6") +__xlated("5: if r12 == 0x0 goto pc+7") __xlated("6: r12 -= 1") -__xlated("7: if r12 != 0x0 goto pc+2") -__xlated("8: r12 = -24") -__xlated("9: call unknown") -__xlated("10: *(u64 *)(r10 -24) = r12") +__xlated("7: if r12 != 0x0 goto pc+3") +__xlated("8: r12 = r10") +__xlated("9: r12 += -24") +__xlated("10: call unknown") +__xlated("11: *(u64 *)(r10 -24) = r12") /* may_goto expansion ends */ -__xlated("11: *(u64 *)(r10 -8) = r1") -__xlated("12: exit") +__xlated("12: *(u64 *)(r10 -8) = r1") +__xlated("13: exit") __success __naked void may_goto_interaction(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index c19b0522df61..2de1ae144ac9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -91,15 +91,16 @@ __arch_loongarch __xlated("0: *(u64 *)(r10 -16) = 65535") __xlated("1: *(u64 *)(r10 -8) = 0") __xlated("2: r12 = *(u64 *)(r10 -16)") -__xlated("3: if r12 == 0x0 goto pc+6") +__xlated("3: if r12 == 0x0 goto pc+7") __xlated("4: r12 -= 1") -__xlated("5: if r12 != 0x0 goto pc+2") -__xlated("6: r12 = -16") -__xlated("7: call unknown") -__xlated("8: *(u64 *)(r10 -16) = r12") -__xlated("9: r0 = 1") -__xlated("10: r0 = 2") -__xlated("11: exit") +__xlated("5: if r12 != 0x0 goto pc+3") +__xlated("6: r12 = r10") +__xlated("7: r12 += -16") +__xlated("8: call unknown") +__xlated("9: *(u64 *)(r10 -16) = r12") +__xlated("10: r0 = 1") +__xlated("11: r0 = 2") +__xlated("12: exit") __success __naked void may_goto_batch_2(void) {