From 4d0f560f4926417f010ce38f8628c41bcad29c9f Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:13 +0000 Subject: [PATCH] bpf: Keep stack arguments in arch_bpf_timed_may_goto() The JITs pass the first stack arguments of a call in registers: x86-64 r9 argument 6 arm64 x5-x7 arguments 6-8 riscv64 a5-a7 arguments 6-8 '*(u64 *)(r11 - 8) = rX' writes such a register in the caller. 'rX = *(u64 *)(r11 + 8)' reads it in the callee. A call changes these registers. Thus the verifier accepts no call between the write and the call of the callee, and no call in the callee before the read. may_goto is not a call for the verifier. But one time in 65535 iterations it calls arch_bpf_timed_may_goto(), and this function calls C code. It saves R0-R5 only. *(u64 *)(r11 - 8) = 0 ; the verifier: argument 6 is 0 call sub sub: 1: may_goto 2f ; C code changes r9 goto 1b 2: r2 = *(u64 *)(r11 + 8) ; r2 is a kernel address r1 = map_value r1 += r2 *(u64 *)(r1 + 0) = 0 On x86-64 the C code changes r9 when the time of may_goto is up. With the hpet clocksource a look at the clock changes r9 too. The prog above crashes the kernel. LLVM does not emit such code: it reads stack arguments at the entry of a function, and it writes them immediately before the call. Save and restore these registers too. The cost is 2 insns on x86-64, 4 on arm64 and 6 on riscv64, one time in 65535 iterations. Fixes: 324c3ca6eed6 ("bpf,x86: Implement JIT support for stack arguments") Fixes: 235b2fe772f5 ("bpf, arm64: Add JIT support for stack arguments") Fixes: ba754185f902 ("riscv, bpf: Add BPF stack arguments support for RV64 JIT") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-19-alexei.starovoitov@gmail.com --- arch/arm64/net/bpf_timed_may_goto.S | 10 ++++++++-- arch/riscv/net/bpf_timed_may_goto.S | 27 ++++++++++++++++++--------- arch/x86/net/bpf_timed_may_goto.S | 3 +++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/arch/arm64/net/bpf_timed_may_goto.S b/arch/arm64/net/bpf_timed_may_goto.S index a9a802711a7f..1b4a4f91ffc9 100644 --- a/arch/arm64/net/bpf_timed_may_goto.S +++ b/arch/arm64/net/bpf_timed_may_goto.S @@ -5,13 +5,16 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* Allocate stack space and emit frame record */ - stp x29, x30, [sp, #-64]! + stp x29, x30, [sp, #-96]! mov x29, sp /* Save BPF registers R0 - R5 (x8, x0-x4)*/ stp x8, x0, [sp, #16] stp x1, x2, [sp, #32] stp x3, x4, [sp, #48] + /* Save stack arguments 6 - 8 (x5-x7) */ + stp x5, x6, [sp, #64] + str x7, [sp, #80] /* * Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP @@ -32,9 +35,12 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) ldp x8, x0, [sp, #16] ldp x1, x2, [sp, #32] ldp x3, x4, [sp, #48] + /* Restore stack arguments 6 - 8 (x5-x7) */ + ldp x5, x6, [sp, #64] + ldr x7, [sp, #80] /* Restore FP and LR */ - ldp x29, x30, [sp], #64 + ldp x29, x30, [sp], #96 ret SYM_FUNC_END(arch_bpf_timed_may_goto) diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index fcf850f244c9..6ec37f634aff 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -10,15 +10,16 @@ * - output: updated count in BPF_REG_AX (t0) * * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved - * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) and stack + * arguments 6-8 (a5-a7) are saved across the call; BPF_REG_FP (s5) is + * callee-saved and needs no saving. */ SYM_FUNC_START(arch_bpf_timed_may_goto) - addi sp, sp, -(8*SZREG) - REG_S ra, 7*SZREG(sp) - REG_S s0, 6*SZREG(sp) - addi s0, sp, 8*SZREG + addi sp, sp, -(12*SZREG) + REG_S ra, 11*SZREG(sp) + REG_S s0, 10*SZREG(sp) + addi s0, sp, 12*SZREG /* Save BPF registers R0-R5 (t6, a0-a4) */ REG_S t6, 5*SZREG(sp) @@ -27,6 +28,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S a2, 2*SZREG(sp) REG_S a3, 1*SZREG(sp) REG_S a4, 0*SZREG(sp) + /* Save stack arguments 6-8 (a5-a7) */ + REG_S a5, 6*SZREG(sp) + REG_S a6, 7*SZREG(sp) + REG_S a7, 8*SZREG(sp) add a0, t0, s5 call bpf_check_timed_may_goto @@ -39,9 +44,13 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_L a1, 3*SZREG(sp) REG_L a0, 4*SZREG(sp) REG_L t6, 5*SZREG(sp) + /* Restore stack arguments 6-8 */ + REG_L a5, 6*SZREG(sp) + REG_L a6, 7*SZREG(sp) + REG_L a7, 8*SZREG(sp) - REG_L s0, 6*SZREG(sp) - REG_L ra, 7*SZREG(sp) - addi sp, sp, 8*SZREG + REG_L s0, 10*SZREG(sp) + REG_L ra, 11*SZREG(sp) + addi sp, sp, 12*SZREG ret SYM_FUNC_END(arch_bpf_timed_may_goto) diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index 6c6e03f725a3..dd2dde89836b 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -22,6 +22,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) pushq %rdx pushq %rcx pushq %r8 + /* Save stack argument 6. */ + pushq %r9 /* * r10 has the pointer to count and timestamp. JIT added the frame @@ -36,6 +38,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* BPF_REG_AX=r10 will be stored into count, so move return value to it. */ movq %rax, %r10 + popq %r9 /* Restore r5-r0. */ popq %r8 popq %rcx