diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d90b6d238b3a..c9419bc65f08 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2626,8 +2626,13 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * Arg 6 goes into r9 register, not on stack. */ outgoing_rsp = out_stack_arg_cnt > 1 ? (out_stack_arg_cnt - 1) * 8 : 0; + /* + * bpf_throw() sees rsp of this frame as it is at a call. The callee-saved + * regs are above the outgoing args, or above r9, which is pushed around + * every call when the stack is private. + */ if (bpf_prog->aux->exception_boundary) - bpf_prog->aux->stack_arg_sp_adjust = outgoing_rsp; + bpf_prog->aux->stack_arg_sp_adjust = outgoing_rsp + (priv_frame_ptr ? 8 : 0); emit_sub_rsp(&prog, outgoing_rsp); if (arena_vm_start) diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index 3d9994a52464..bbc09f516d92 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -279,6 +279,105 @@ int private_stack_exception_sub_prog(void) return 0; } +/* + * Nothing attaches the two fentry progs above, so they are loaded, but never + * run. BPF_PROG_TEST_RUN does run a raw_tp prog. The default exception callback + * returns the cookie of bpf_throw(). + */ +SEC("raw_tp") +__description("Private stack, run, exception in main prog") +__success __retval(1) +__arch_x86_64 +__jited(" movq %rdi, -0x200(%r9)") +__jited(" movl $0x1, %edi") +__jited(" pushq %r9") +__jited("...") +__jited(" callq") +__naked void private_stack_run_exception_main_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 1; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +__used +__naked static void subprog_throw(void) +{ + asm volatile (" \ + r1 = 2; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +SEC("raw_tp") +__description("Private stack, run, exception in subprog") +__success __retval(2) +__arch_x86_64 +__jited(" movq %rdi, -0x200(%r9)") +__jited(" pushq %r9") +__jited("...") +__jited(" callq") +__jited(" popq %r9") +__naked void private_stack_run_exception_sub_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + call subprog_throw; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__used +__naked static void private_stack_subprog_throw(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 3; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +/* Only the frame of the main prog counts, and it has nothing pushed at the call */ +SEC("raw_tp") +__description("Private stack, run, exception in subprog, no private stack in main prog") +__success __retval(3) +__arch_x86_64 +__jited(" subq $0x8, %rsp") +__jited("...") +__jited("func #1") +__jited("...") +__jited(" movq %rdi, -0x200(%r9)") +__jited(" movl $0x3, %edi") +__jited(" pushq %r9") +__naked void no_private_stack_run_exception_sub_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 8) = r1; \ + call private_stack_subprog_throw; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + int glob; __noinline static void subprog2(int *val) {