github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: Keep stack arguments in arch_bpf_timed_may_goto()

Browse Source 
The JITs pass the first stack arguments of a call in registers:

  x86-64    r9       argument 6
  arm64     x5-x7    arguments 6-8
  riscv64   a5-a7    arguments 6-8

'*(u64 *)(r11 - 8) = rX' writes such a register in the caller.
'rX = *(u64 *)(r11 + 8)' reads it in the callee. A call changes these
registers. Thus the verifier accepts no call between the write and the call
of the callee, and no call in the callee before the read.

may_goto is not a call for the verifier. But one time in 65535 iterations
it calls arch_bpf_timed_may_goto(), and this function calls C code. It
saves R0-R5 only.

       *(u64 *)(r11 - 8) = 0      ; the verifier: argument 6 is 0
       call sub
  sub:
    1: may_goto 2f                ; C code changes r9
       goto 1b
    2: r2 = *(u64 *)(r11 + 8)     ; r2 is a kernel address
       r1 = map_value
       r1 += r2
       *(u64 *)(r1 + 0) = 0

On x86-64 the C code changes r9 when the time of may_goto is up. With the
hpet clocksource a look at the clock changes r9 too. The prog above crashes
the kernel. LLVM does not emit such code: it reads stack arguments at the
entry of a function, and it writes them immediately before the call.

Save and restore these registers too. The cost is 2 insns on x86-64, 4 on
arm64 and 6 on riscv64, one time in 65535 iterations.

Fixes: 324c3ca6ee ("bpf,x86: Implement JIT support for stack arguments")
Fixes: 235b2fe772 ("bpf, arm64: Add JIT support for stack arguments")
Fixes: ba754185f9 ("riscv, bpf: Add BPF stack arguments support for RV64 JIT")
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://patch.msgid.link/20261027190314.1440199-19-alexei.starovoitov@gmail.com
This commit is contained in:
Alexei Starovoitov 2026-10-27 19:03:13 +00:00
parent df9e4a87a8
commit 4d0f560f49
No known key found for this signature in database
3 changed files with 29 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
arch/arm64/net/bpf_timed_may_goto.S
View File

@ -5,13 +5,16 @@
SYM_FUNC_START(arch_bpf_timed_may_goto)
/* Allocate stack space and emit frame record */
stp x29, x30, [sp, #-64]!
stp x29, x30, [sp, #-96]!
mov x29, sp
/* Save BPF registers R0 - R5 (x8, x0-x4)*/
stp x8, x0, [sp, #16]
stp x1, x2, [sp, #32]
stp x3, x4, [sp, #48]
/* Save stack arguments 6 - 8 (x5-x7) */
stp x5, x6, [sp, #64]
str x7, [sp, #80]
/*
* Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP
@ -32,9 +35,12 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
ldp x8, x0, [sp, #16]
ldp x1, x2, [sp, #32]
ldp x3, x4, [sp, #48]
/* Restore stack arguments 6 - 8 (x5-x7) */
ldp x5, x6, [sp, #64]
ldr x7, [sp, #80]
/* Restore FP and LR */
ldp x29, x30, [sp], #64
ldp x29, x30, [sp], #96
ret
SYM_FUNC_END(arch_bpf_timed_may_goto)

27
arch/riscv/net/bpf_timed_may_goto.S
View File

@ -10,15 +10,16 @@
* - output: updated count in BPF_REG_AX (t0)
*
* Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where
* ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved
* across the call; BPF_REG_FP (s5) is callee-saved and needs no saving.
* ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) and stack
* arguments 6-8 (a5-a7) are saved across the call; BPF_REG_FP (s5) is
* callee-saved and needs no saving.
*/
SYM_FUNC_START(arch_bpf_timed_may_goto)
addi sp, sp, -(8*SZREG)
REG_S ra, 7*SZREG(sp)
REG_S s0, 6*SZREG(sp)
addi s0, sp, 8*SZREG
addi sp, sp, -(12*SZREG)
REG_S ra, 11*SZREG(sp)
REG_S s0, 10*SZREG(sp)
addi s0, sp, 12*SZREG
/* Save BPF registers R0-R5 (t6, a0-a4) */
REG_S t6, 5*SZREG(sp)
@ -27,6 +28,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
REG_S a2, 2*SZREG(sp)
REG_S a3, 1*SZREG(sp)
REG_S a4, 0*SZREG(sp)
/* Save stack arguments 6-8 (a5-a7) */
REG_S a5, 6*SZREG(sp)
REG_S a6, 7*SZREG(sp)
REG_S a7, 8*SZREG(sp)
add a0, t0, s5
call bpf_check_timed_may_goto
@ -39,9 +44,13 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
REG_L a1, 3*SZREG(sp)
REG_L a0, 4*SZREG(sp)
REG_L t6, 5*SZREG(sp)
/* Restore stack arguments 6-8 */
REG_L a5, 6*SZREG(sp)
REG_L a6, 7*SZREG(sp)
REG_L a7, 8*SZREG(sp)
REG_L s0, 6*SZREG(sp)
REG_L ra, 7*SZREG(sp)
addi sp, sp, 8*SZREG
REG_L s0, 10*SZREG(sp)
REG_L ra, 11*SZREG(sp)
addi sp, sp, 12*SZREG
ret
SYM_FUNC_END(arch_bpf_timed_may_goto)

3
arch/x86/net/bpf_timed_may_goto.S
View File

@ -22,6 +22,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
pushq %rdx
pushq %rcx
pushq %r8
/* Save stack argument 6. */
pushq %r9
/*
* r10 has the pointer to count and timestamp. JIT added the frame
@ -36,6 +38,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
/* BPF_REG_AX=r10 will be stored into count, so move return value to it. */
movq %rax, %r10
popq %r9
/* Restore r5-r0. */
popq %r8
popq %rcx