From 91068ed2d3ee0713d7566c7977b995f8a0936bea Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 31 Oct 2026 05:05:57 +0000 Subject: [PATCH 1/2] bpf, x86: Fix bpf_throw() in a prog with a private stack When the main prog has a private stack and the prog calls bpf_throw(), it returns to the kernel with wrong values in r12, rbx, r13, r14 and r15. The kernel crashes soon after that, for example: BUG: kernel NULL pointer dereference, address: 0000000000000010 RIP: 0010:__bpf_prog_test_run_raw_tp+0x125/0x380 The main prog of a prog that can throw saves these five registers at its entry. bpf_throw() walks the stack to the frame of the main prog and gives the stack pointer of that frame to the exception callback, which pops the registers from there. In a function with a private stack r9 is the frame pointer, and the JIT emits 'push r9' in front of every call and 'pop r9' after it. So the stack pointer that bpf_throw() finds is 8 bytes too low: r12 rbx r13 r14 r15 <- the callback has to pop from here r9 <- rsp of the main prog at the call r15 gets the address of the private stack and every other register gets the value of its neighbour. This happens in tracepoint, raw_tp, perf_event and tracing progs, except iterators, and in struct_ops progs that ask for a private stack, when the main prog has more than 48 bytes of stack. The throw can be in the main prog or in a subprog. aux->stack_arg_sp_adjust already moves the stack pointer from the outgoing stack arguments up to the saved registers. Add the 8 bytes of r9 to it when the main prog has a private stack. A constant is enough. Calls of helpers, kfuncs and subprogs and callx all have the push. A prog with tail calls has no private stack, and a callback cannot throw. Only the frame of the main prog matters: the frames of subprogs are dropped, with or without a private stack. On x86-64 a function with stack arguments has no private stack, hence only one of the two parts is not zero. The JITed code does not change. Fixes: 7d1cd70d4b16 ("bpf, x86: Support private stack in jit") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261031050558.2410721-2-alexei.starovoitov@gmail.com --- arch/x86/net/bpf_jit_comp.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d90b6d238b3a..c9419bc65f08 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2626,8 +2626,13 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * Arg 6 goes into r9 register, not on stack. */ outgoing_rsp = out_stack_arg_cnt > 1 ? (out_stack_arg_cnt - 1) * 8 : 0; + /* + * bpf_throw() sees rsp of this frame as it is at a call. The callee-saved + * regs are above the outgoing args, or above r9, which is pushed around + * every call when the stack is private. + */ if (bpf_prog->aux->exception_boundary) - bpf_prog->aux->stack_arg_sp_adjust = outgoing_rsp; + bpf_prog->aux->stack_arg_sp_adjust = outgoing_rsp + (priv_frame_ptr ? 8 : 0); emit_sub_rsp(&prog, outgoing_rsp); if (arena_vm_start) From 6cc0cf0a6fb087f1979dfad5c1e13a41e57f55d7 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Sat, 31 Oct 2026 05:05:58 +0000 Subject: [PATCH 2/2] selftests/bpf: Test bpf_throw() in a prog with a private stack 'Private stack, exception in main prog' and 'Private stack, exception in subprog' are fentry progs. The test loader does not attach them, and BPF_PROG_TEST_RUN of a fentry prog does not call the prog. They are loaded, but they never run. Add three raw_tp progs. BPF_PROG_TEST_RUN runs them, and the default exception callback returns the cookie of bpf_throw(): - the main prog has a private stack and throws - the main prog has a private stack, its subprog throws - the main prog has no private stack, its subprog has one and throws Without the fix of the x86-64 JIT the first two crash the kernel. The third one passes with and without it: it makes sure that only the frame of the main prog is taken into account. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261031050558.2410721-3-alexei.starovoitov@gmail.com --- .../bpf/progs/verifier_private_stack.c | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index 3d9994a52464..bbc09f516d92 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -279,6 +279,105 @@ int private_stack_exception_sub_prog(void) return 0; } +/* + * Nothing attaches the two fentry progs above, so they are loaded, but never + * run. BPF_PROG_TEST_RUN does run a raw_tp prog. The default exception callback + * returns the cookie of bpf_throw(). + */ +SEC("raw_tp") +__description("Private stack, run, exception in main prog") +__success __retval(1) +__arch_x86_64 +__jited(" movq %rdi, -0x200(%r9)") +__jited(" movl $0x1, %edi") +__jited(" pushq %r9") +__jited("...") +__jited(" callq") +__naked void private_stack_run_exception_main_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 1; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +__used +__naked static void subprog_throw(void) +{ + asm volatile (" \ + r1 = 2; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +SEC("raw_tp") +__description("Private stack, run, exception in subprog") +__success __retval(2) +__arch_x86_64 +__jited(" movq %rdi, -0x200(%r9)") +__jited(" pushq %r9") +__jited("...") +__jited(" callq") +__jited(" popq %r9") +__naked void private_stack_run_exception_sub_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + call subprog_throw; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__used +__naked static void private_stack_subprog_throw(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 3; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +/* Only the frame of the main prog counts, and it has nothing pushed at the call */ +SEC("raw_tp") +__description("Private stack, run, exception in subprog, no private stack in main prog") +__success __retval(3) +__arch_x86_64 +__jited(" subq $0x8, %rsp") +__jited("...") +__jited("func #1") +__jited("...") +__jited(" movq %rdi, -0x200(%r9)") +__jited(" movl $0x3, %edi") +__jited(" pushq %r9") +__naked void no_private_stack_run_exception_sub_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 8) = r1; \ + call private_stack_subprog_throw; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + int glob; __noinline static void subprog2(int *val) {