From 5bbbce02e500d47d8e259a45be5a7be9741d0533 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Fri, 14 Aug 2026 15:02:53 -0700 Subject: [PATCH 1/2] bpf, x86: Fix per-CPU address resolution into an extended register The destination of the per-CPU address MOV is encoded in ModRM.reg, which is extended by REX.R, but the REX prefix is built with add_1mod(), which sets REX.B. REX.B extends ModRM.rm and SIB.base, and this instruction addresses memory as disp32 with no base, so the bit has no effect at all and the high register bit is simply lost. Every is_ereg() destination therefore resolves to the wrong register, picking whichever one shares the low three bits: R5 -> RAX R7 -> RBP R8 -> RSI R9 -> RDI With BPF_REG_5, whose reg2hex is 0, the emitted 65 49 03 04 25 add %gs:,%rax adds the per-CPU offset to RAX rather than R8. The destination keeps the unadjusted address and RAX is clobbered, so the program goes on to dereference a pointer that was never made per-CPU: BUG: unable to handle page fault for address: 0000607e386a8894 RIP: bpf_prog_707837aafd2aa9ae_update_percpu_data+0x93/0xc9 Call Trace: __bpf_prog_test_run_raw_tp+0x2dc/0x7d0 __flush_smp_call_function_queue+0x1e9/0xc80 Kernel panic - not syncing: Fatal exception in interrupt R5 is the mildest of the four, aliasing a scratch register and faulting at the store. R7 aliases RBP and would corrupt the frame pointer, R8 and R9 alias the argument registers. Use add_2mod() so the register goes through REX.R, matching how add_2reg() places it in ModRM.reg and how emit_priv_frame_ptr() hardcodes 0x4c for the same instruction with R9. Encodings for the non-extended registers are unchanged. Problem showed up when trying to resurrect BPF_GCC CI (selftests built with BPF_GCC). This has gone unnoticed because clang reloads the address into R1 before each per-CPU access, so the destination is never an extended register. GCC keeps several per-CPU addresses live at once, and test_progs-bpf_gcc panics the kernel in global_percpu_data/init, where the address of a .percpu variable ends up in R5. Fixes: 7bdbf7446305 ("bpf: add special internal-only MOV instruction to resolve per-CPU addrs") Signed-off-by: Vineet Gupta Reviewed-by: Eduard Zingerman Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260814220254.3797467-2-vineet.gupta@linux.dev Signed-off-by: Eduard Zingerman --- arch/x86/net/bpf_jit_comp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d920772af7d5..1a9fb530adc3 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1935,7 +1935,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_mov(dst_reg, src_reg); #ifdef CONFIG_SMP /* add , gs:[] */ - EMIT2(0x65, add_1mod(0x48, dst_reg)); + EMIT2(0x65, add_2mod(0x48, 0, dst_reg)); EMIT3(0x03, add_2reg(0x04, 0, dst_reg), 0x25); EMIT((u32)(unsigned long)&this_cpu_off, 4); #endif From f61306e8c98ce63021efb3091d422260b6be25bb Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Fri, 14 Aug 2026 15:02:54 -0700 Subject: [PATCH 2/2] selftests/bpf: Check per-CPU address resolution per register An ld_imm64 of a per-CPU map value is followed by a mov_percpu_addr that reuses the same register, so which register the address lands in decides how the JIT encodes the add. Getting the REX prefix wrong there is invisible to a functional test unless the address happens to land in an extended register, which is why this went unnoticed. Load a .percpu variable into every register in one program and match the JITed add against the register each one must resolve into. Signed-off-by: Vineet Gupta Link: https://patch.msgid.link/20260814220254.3797467-3-vineet.gupta@linux.dev Signed-off-by: Eduard Zingerman --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_percpu_addr.c | 72 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_percpu_addr.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 8113fea7ba86..64ac49ad67e6 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -79,6 +79,7 @@ #include "verifier_netfilter_retcode.skel.h" #include "verifier_bpf_fastcall.skel.h" #include "verifier_or_jmp32_k.skel.h" +#include "verifier_percpu_addr.skel.h" #include "verifier_precision.skel.h" #include "verifier_prevent_map_lookup.skel.h" #include "verifier_private_stack.skel.h" @@ -240,6 +241,7 @@ void test_verifier_netfilter_ctx(void) { RUN(verifier_netfilter_ctx); } void test_verifier_netfilter_retcode(void) { RUN(verifier_netfilter_retcode); } void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); } void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); } +void test_verifier_percpu_addr(void) { RUN(verifier_percpu_addr); } void test_verifier_precision(void) { RUN(verifier_precision); } void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); } void test_verifier_private_stack(void) { RUN(verifier_private_stack); } diff --git a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c new file mode 100644 index 000000000000..967f4e6e3a49 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "bpf_misc.h" + +#if defined(__TARGET_ARCH_x86) + +int percpu_data SEC(".percpu"); + +/* + * An ld_imm64 of a per-CPU map value is followed by a mov_percpu_addr that + * reuses the same register, so check that the add resolves into the register + * the address was loaded into, for every register. + */ +SEC("raw_tp") +__description("per-CPU address resolution") +__success +__arch_x86_64 +__jited(" movabsq $0x{{.*}}, %rax") +__jited(" addq %gs:{{.*}}, %rax") +__jited(" movabsq $0x{{.*}}, %rdi") +__jited(" addq %gs:{{.*}}, %rdi") +__jited(" movabsq $0x{{.*}}, %rsi") +__jited(" addq %gs:{{.*}}, %rsi") +__jited(" movabsq $0x{{.*}}, %rdx") +__jited(" addq %gs:{{.*}}, %rdx") +__jited(" movabsq $0x{{.*}}, %rcx") +__jited(" addq %gs:{{.*}}, %rcx") +__jited(" movabsq $0x{{.*}}, %r8") +__jited(" addq %gs:{{.*}}, %r8") +__jited(" movabsq $0x{{.*}}, %rbx") +__jited(" addq %gs:{{.*}}, %rbx") +__jited(" movabsq $0x{{.*}}, %r13") +__jited(" addq %gs:{{.*}}, %r13") +__jited(" movabsq $0x{{.*}}, %r14") +__jited(" addq %gs:{{.*}}, %r14") +__jited(" movabsq $0x{{.*}}, %r15") +__jited(" addq %gs:{{.*}}, %r15") +__naked void percpu_addr(void) +{ + asm volatile (" \ + r0 = %[percpu_data] ll; \ + r1 = %[percpu_data] ll; \ + r2 = %[percpu_data] ll; \ + r3 = %[percpu_data] ll; \ + r4 = %[percpu_data] ll; \ + r5 = %[percpu_data] ll; \ + r6 = %[percpu_data] ll; \ + r7 = %[percpu_data] ll; \ + r8 = %[percpu_data] ll; \ + r9 = %[percpu_data] ll; \ + r0 = 0; \ + exit; \ +" : + : __imm_addr(percpu_data) + : __clobber_all); +} + +#else + +SEC("raw_tp") +__description("percpu addr dummy") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL";