Merge branch 'bpf-x86-fix-per-cpu-address-resolution-into-an-extended-register'

Vineet Gupta says:

====================
bpf, x86: fix per-CPU address resolution into an extended register

The JIT resolves a per-CPU address with

  add <dst>, gs:[this_cpu_off]

but builds the REX prefix with add_1mod(), which sets REX.B. The
destination is encoded in ModRM.reg, which REX.R extends, and the memory
operand is disp32 with no base, so REX.B does nothing and the high
register bit is dropped. Every extended destination therefore resolves
into whichever register shares the low three bits:

  R5 -> RAX    R7 -> RBP    R8 -> RSI    R9 -> RDI

The address is left unadjusted and an unrelated register is clobbered.
Patch 1 switches to add_2mod() so the bit goes through REX.R.

Clang reloads the address into R1 before each per-CPU access, so the
destination is never an extended register and the bug has been dormant
since v6.10. GCC keeps several per-CPU addresses live at once, which is
how it turned up: test_progs-bpf_gcc panics the kernel in
global_percpu_data/init, with the address of a .percpu variable in R5.

Patch 2 covers every register. A functional test only catches this if
the address happens to land in an extended register, so the test matches
the JITed add instead.

Changes in v3:
- Fold the five per-register programs into one that loads every
  register, and drop the comment explaining the register choice
  (Eduard Zingerman).
- Move the percpu_data declaration inside the arch guard, so other
  targets no longer carry a .percpu section and an unused map (bpf-ci).
- Match the movabsq of each address as well as the add, so the matchers
  stay on consecutive lines and the pair is checked to use the same
  register.
- Restore the Reviewed-by on patch 1, dropped by mistake in v2.

Changes in v2:
- Add the selftest, patch 2/2 (Eduard Zingerman). It uses __jited()
  rather than __xlated(): the xlated stream is identical for every
  register, and the wrong prefix is only visible in the native encoding.
- No functional change to patch 1.
====================

Link: https://patch.msgid.link/20260814220254.3797467-1-vineet.gupta@linux.dev
Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>

arch/x86/net/bpf_jit_comp.c

@@ -1935,7 +1935,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 				EMIT_mov(dst_reg, src_reg);
 #ifdef CONFIG_SMP
 				/* add <dst>, gs:[<off>] */
-				EMIT2(0x65, add_1mod(0x48, dst_reg));
+				EMIT2(0x65, add_2mod(0x48, 0, dst_reg));
 				EMIT3(0x03, add_2reg(0x04, 0, dst_reg), 0x25);
 				EMIT((u32)(unsigned long)&this_cpu_off, 4);
 #endif

tools/testing/selftests/bpf/prog_tests/verifier.c

@@ -79,6 +79,7 @@
 #include "verifier_netfilter_retcode.skel.h"
 #include "verifier_bpf_fastcall.skel.h"
 #include "verifier_or_jmp32_k.skel.h"
+#include "verifier_percpu_addr.skel.h"
 #include "verifier_precision.skel.h"
 #include "verifier_prevent_map_lookup.skel.h"
 #include "verifier_private_stack.skel.h"
@@ -240,6 +241,7 @@ void test_verifier_netfilter_ctx(void)        { RUN(verifier_netfilter_ctx); }
 void test_verifier_netfilter_retcode(void)    { RUN(verifier_netfilter_retcode); }
 void test_verifier_bpf_fastcall(void)         { RUN(verifier_bpf_fastcall); }
 void test_verifier_or_jmp32_k(void)           { RUN(verifier_or_jmp32_k); }
+void test_verifier_percpu_addr(void)          { RUN(verifier_percpu_addr); }
 void test_verifier_precision(void)            { RUN(verifier_precision); }
 void test_verifier_prevent_map_lookup(void)   { RUN(verifier_prevent_map_lookup); }
 void test_verifier_private_stack(void)        { RUN(verifier_private_stack); }

tools/testing/selftests/bpf/progs/verifier_percpu_addr.c

@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+#if defined(__TARGET_ARCH_x86)
+
+int percpu_data SEC(".percpu");
+
+/*
+ * An ld_imm64 of a per-CPU map value is followed by a mov_percpu_addr that
+ * reuses the same register, so check that the add resolves into the register
+ * the address was loaded into, for every register.
+ */
+SEC("raw_tp")
+__description("per-CPU address resolution")
+__success
+__arch_x86_64
+__jited("	movabsq	$0x{{.*}}, %rax")
+__jited("	addq	%gs:{{.*}}, %rax")
+__jited("	movabsq	$0x{{.*}}, %rdi")
+__jited("	addq	%gs:{{.*}}, %rdi")
+__jited("	movabsq	$0x{{.*}}, %rsi")
+__jited("	addq	%gs:{{.*}}, %rsi")
+__jited("	movabsq	$0x{{.*}}, %rdx")
+__jited("	addq	%gs:{{.*}}, %rdx")
+__jited("	movabsq	$0x{{.*}}, %rcx")
+__jited("	addq	%gs:{{.*}}, %rcx")
+__jited("	movabsq	$0x{{.*}}, %r8")
+__jited("	addq	%gs:{{.*}}, %r8")
+__jited("	movabsq	$0x{{.*}}, %rbx")
+__jited("	addq	%gs:{{.*}}, %rbx")
+__jited("	movabsq	$0x{{.*}}, %r13")
+__jited("	addq	%gs:{{.*}}, %r13")
+__jited("	movabsq	$0x{{.*}}, %r14")
+__jited("	addq	%gs:{{.*}}, %r14")
+__jited("	movabsq	$0x{{.*}}, %r15")
+__jited("	addq	%gs:{{.*}}, %r15")
+__naked void percpu_addr(void)
+{
+	asm volatile ("					\
+	r0 = %[percpu_data] ll;				\
+	r1 = %[percpu_data] ll;				\
+	r2 = %[percpu_data] ll;				\
+	r3 = %[percpu_data] ll;				\
+	r4 = %[percpu_data] ll;				\
+	r5 = %[percpu_data] ll;				\
+	r6 = %[percpu_data] ll;				\
+	r7 = %[percpu_data] ll;				\
+	r8 = %[percpu_data] ll;				\
+	r9 = %[percpu_data] ll;				\
+	r0 = 0;						\
+	exit;						\
+"	:
+	: __imm_addr(percpu_data)
+	: __clobber_all);
+}
+
+#else
+
+SEC("raw_tp")
+__description("percpu addr dummy")
+__success
+int dummy_test(void)
+{
+	return 0;
+}
+
+#endif
+
+char _license[] SEC("license") = "GPL";