From 2e320ea9a883b307bed2a7a54af96ca0b902f902 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 20 Oct 2026 23:19:39 +0000 Subject: [PATCH 1/3] bpf: Make unwind_exit of an exception callback exit in JITs For the verifier unwind_exit of an exception callback is exit. For JITs the callback is a subprog, so they emit the lookup of the return address. The idea was that the kernel calls the callback and the lookup finds nothing. But an exception callback has no frame of its own. It returns from the frame of the main prog that threw, and the kernel isn't always the caller of that one: - An extension returns to the caller of the subprog that it replaced. - After a tail call in a subprog the main prog returns to the caller of that subprog. When unwind_goto follows that call, the callback goes to the pad, and the pad wasn't verified with it: r6 = 1 call a // can unwind unwind_goto pad r6 = 2 call b // cannot unwind unwind_goto pad // stays, since the pad is not dead code exit pad: r0 = r6 // r6 is 1 for the verifier An extension without unwind_exit can replace b(). When it throws and its exception callback ends with unwind_exit, the pad runs with r6 = 2. Both JITs ask bpf_prog_can_unwind(). Let it say no for an exception callback. An async callback has a frame of its own and the kernel is the caller. Fixes: a2d274771321 ("bpf: Add unwind_goto and unwind_exit insns") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/5f24cbed1ec4a6a0ecd0b834d738c48dc9eb64362be4fa50dfb0eaddfad928b3@mail.kernel.org/ Link: https://patch.msgid.link/20261020231942.2568163-2-alexei.starovoitov@gmail.com Closes: https://lore.kernel.org/bpf/sashiko-outbox-179498@kernel.org/ --- arch/arm64/net/bpf_jit_comp.c | 2 +- arch/x86/net/bpf_jit_comp.c | 2 +- include/linux/filter.h | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index b3efe0668e86..326e9e1dea0a 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1273,7 +1273,7 @@ static long bpf_unwind_fixup(u64 lr) e = search_bpf_extables(ptrauth_strip_kernel_insn_pac(lr) - 1); /* * The verifier made sure that unwind_goto follows a call in a prog. When - * the kernel called an async or an exception callback, it's exit. + * the kernel called an async callback, it's exit. */ return e ? e->fixup : 0; } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d3ab36281144..6a2fe3d0039d 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2552,7 +2552,7 @@ static u64 bpf_unwind_target(u64 ret_addr) /* * The verifier made sure that unwind_goto follows a call in a prog. When - * the kernel called an async or an exception callback, it's exit. + * the kernel called an async callback, it's exit. */ return e ? ret_addr + e->fixup : ret_addr; } diff --git a/include/linux/filter.h b/include/linux/filter.h index fccd8db33647..e0218e677a1d 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -267,9 +267,13 @@ static inline bool bpf_is_unwind_exit(const struct bpf_insn *insn) /* * The caller of the main prog doesn't tell unwind_exit from exit. An extension * is different: its caller is the caller of the subprog that it replaced. + * An exception callback returns from the frame of the main prog. After freplace + * or a tail call it returns to a prog that wasn't verified with the callback. */ static inline bool bpf_prog_can_unwind(const struct bpf_prog *prog) { + if (prog->aux->exception_cb) + return false; return bpf_is_subprog(prog) || prog->type == BPF_PROG_TYPE_EXT; } From 9b27cd0554c5883318256c32b2aaf060a3e20e0a Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 20 Oct 2026 23:19:40 +0000 Subject: [PATCH 2/3] bpf: Reject unwind_goto and unwind_exit in offloaded progs A prog with unwind_goto or unwind_exit has to be JITed by a JIT that supports them. The driver JITs an offloaded prog, and nobody looks at jit_required there. nfp rejects unwind_goto in nfp_verify_insn(). netdevsim takes everything. Nothing runs such a prog, but it's not jited for bpf_insn_prepare_dump(), which takes unwind_goto for BPF_CALL_ARGS and shows it as a call. Fixes: a2d274771321 ("bpf: Add unwind_goto and unwind_exit insns") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261020231942.2568163-3-alexei.starovoitov@gmail.com Closes: https://lore.kernel.org/bpf/sashiko-outbox-179499@kernel.org/ --- kernel/bpf/verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7d7193b087f6..5601d2056218 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3462,7 +3462,7 @@ static int check_unwind_insns(struct bpf_verifier_env *env) } if (!seen) return 0; - if (!bpf_jit_supports_unwind()) { + if (!bpf_jit_supports_unwind() || bpf_prog_is_offloaded(env->prog->aux)) { verbose(env, "JIT does not support unwinding\n"); return -ENOTSUPP; } From 4c2cdb6156920b5fb578779b5da62edf3fc45556 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 20 Oct 2026 23:19:41 +0000 Subject: [PATCH 3/3] selftests/bpf: Test unwind_exit of an exception callback In both tests the prog that throws returns to a call that unwind_goto follows, and its exception callback ends with unwind_exit: - An extension replaces a subprog that cannot unwind. unwind_goto after the call stays, since it shares the pad with a call of a subprog that can unwind. - A subprog that can unwind does a tail call. unwind_exit of the callback has to be exit. Without the fix the callers go to the pad and return 2 instead of 3 and 42 instead of 7. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261020231942.2568163-4-alexei.starovoitov@gmail.com --- .../selftests/bpf/prog_tests/unwind_attach.c | 22 ++++++++++++-- .../selftests/bpf/progs/unwind_attach.c | 29 +++++++++++++++++-- .../selftests/bpf/progs/verifier_unwind.c | 29 +++++++++++++++++++ 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/unwind_attach.c b/tools/testing/selftests/bpf/prog_tests/unwind_attach.c index ada8b3273ccb..37e03b3e0ee8 100644 --- a/tools/testing/selftests/bpf/prog_tests/unwind_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/unwind_attach.c @@ -67,7 +67,7 @@ void test_unwind_attach(void) { struct verifier_unwind *tgt; struct bpf_program *prog; - int tgt_fd, plain_fd, err; + int tgt_fd, plain_fd, shared_fd, tail_fd, throws_fd, err, key = 0; if (!is_jit_enabled()) { printf("%s:SKIP:JIT is disabled\n", __func__); @@ -79,7 +79,10 @@ void test_unwind_attach(void) return; bpf_object__for_each_program(prog, tgt->obj) bpf_program__set_autoload(prog, prog == tgt->progs.calls_global_unwinds || - prog == tgt->progs.calls_global_never_unwinds); + prog == tgt->progs.calls_global_never_unwinds || + prog == tgt->progs.shared_pad || + prog == tgt->progs.tail_call_in_subprog || + prog == tgt->progs.exception_callback_unwinds); err = verifier_unwind__load(tgt); if (err == -ENOTSUPP) { printf("%s:SKIP:JIT does not support unwinding\n", __func__); @@ -107,6 +110,21 @@ void test_unwind_attach(void) attach(plain_fd, "global_never_unwinds", "freplace_unwinds", "Extension program can unwind, while original does not", 0); ASSERT_EQ(run(tgt_fd), 15, "retval"); + + /* + * An exception callback returns from the frame of the main prog. Here + * that's a return to a call that unwind_goto follows. unwind_exit of + * the callback has to be exit: the pad wasn't verified with it. + */ + shared_fd = bpf_program__fd(tgt->progs.shared_pad); + attach(shared_fd, "global_never_unwinds", "freplace_throws", NULL, 3); + + tail_fd = bpf_program__fd(tgt->progs.tail_call_in_subprog); + throws_fd = bpf_program__fd(tgt->progs.exception_callback_unwinds); + err = bpf_map__update_elem(tgt->maps.prog_array, &key, sizeof(key), + &throws_fd, sizeof(throws_fd), 0); + if (ASSERT_OK(err, "prog_array")) + ASSERT_EQ(run(tail_fd), 7, "retval"); out: verifier_unwind__destroy(tgt); } diff --git a/tools/testing/selftests/bpf/progs/unwind_attach.c b/tools/testing/selftests/bpf/progs/unwind_attach.c index df73c26c1ff4..9aff7c6bda23 100644 --- a/tools/testing/selftests/bpf/progs/unwind_attach.c +++ b/tools/testing/selftests/bpf/progs/unwind_attach.c @@ -7,6 +7,13 @@ /* The targets are global_unwinds() and global_never_unwinds() in verifier_unwind.c */ +#define unwind_exit ".8byte %[insn_unwind_exit];" +#define __imm_unwind_exit \ + __imm_insn(insn_unwind_exit, \ + BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0)) + +extern void bpf_throw(__u64 cookie) __ksym; + int fentry_cnt; SEC("?fentry/global_unwinds") @@ -33,11 +40,27 @@ __naked unsigned long freplace_unwinds(void) { asm volatile ( "r0 = 3;" - ".8byte %[unwind_exit];" + unwind_exit : - : __imm_insn(unwind_exit, - BPF_RAW_INSN(BPF_JMP | BPF_EXIT, 0, BPF_UNWIND_EXIT, 0, 0)) + : __imm_unwind_exit ); } +/* Not naked: the argument has to have a name in BTF */ +__noinline int throws_cb(__u64 cookie) +{ + if (cookie) + asm volatile ("r0 = 3;" unwind_exit :: __imm_unwind_exit : "r0"); + return 0; +} + +/* It has no unwind_exit, so it can replace a subprog that cannot unwind */ +SEC("?freplace/global_never_unwinds") +__attribute__((btf_decl_tag("exception_callback:throws_cb"))) +unsigned long freplace_throws(void) +{ + bpf_throw(1); + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_unwind.c b/tools/testing/selftests/bpf/progs/verifier_unwind.c index 0a75cb8af67f..5a21345c34d4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_unwind.c +++ b/tools/testing/selftests/bpf/progs/verifier_unwind.c @@ -611,6 +611,35 @@ __naked int calls_global_never_unwinds(void) ); } +/* + * The verifier gets to the pad from the first call only, with r6 = 1. The second + * unwind_goto stays, since the pad is not dead code. + */ +SEC("tc") +__description("unwind: a pad shared with a call of a subprog that cannot unwind") +__arch_x86_64 +__arch_arm64 +__load_if_JITed() +__success __retval(7) +__naked int shared_pad(void) +{ + asm volatile ( + "r6 = 1;" + "call global_returns;" + unwind_goto(4) + "r6 = 2;" + "call global_never_unwinds;" + unwind_goto(1) + "exit;" + "r0 = r6;" + "exit;" + : + : __imm_unwind_goto(4), + __imm_unwind_goto(1) + : __clobber_all + ); +} + SEC("tc") __description("unwind: no unwind_goto after a call of a global subprog") __arch_x86_64