Merge branch 'signed-loads-from-arena'

Puranjay Mohan says: ==================== Signed loads from Arena Changelog: v3 -> v4: v3: https://lore.kernel.org/all/20250915162848.54282-1-puranjay@kernel.org/ - Update bpf_jit_supports_insn() in riscv jit to reject signed arena loads (Eduard) - Fix coding style related to braces usage in an if statement in x86 jit (Eduard) v2 -> v3: v2: https://lore.kernel.org/bpf/20250514175415.2045783-1-memxor@gmail.com/ - Fix encoding for the generated instructions in x86 JIT (Eduard) The patch in v2 was generating instructions like: 42 63 44 20 f8 movslq -0x8(%rax,%r12), %eax This doesn't make sense because movslq outputs a 64-bit result, but the destination register here is set to eax (32-bit). The fix it to set the REX.W bit in the opcode, that means changing EMIT2(add_3mod(0x40, ...)) to EMIT2(add_3mod(0x48, ...)) - Add arm64 support - Add selftests signed laods from arena. v1 -> v2: v1: https://lore.kernel.org/bpf/20250509194956.1635207-1-memxor@gmail.com - Use bpf_jit_supports_insn. (Alexei) Currently, signed load instructions into arena memory are unsupported. The compiler is free to generate these, and on GCC-14 we see a corresponding error when it happens. The hurdle in supporting them is deciding which unused opcode to use to mark them for the JIT's own consumption. After much thinking, it appears 0xc0 / BPF_NOSPEC can be combined with load instructions to identify signed arena loads. Use this to recognize and JIT them appropriately, and remove the verifier side limitation on the program if the JIT supports them. ==================== Link: https://patch.msgid.link/20250923110157.18326-1-puranjay@kernel.org Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2026-05-24 15:12:13 +02:00 · 2025-09-23 12:00:23 -07:00 · 2025-09-23 12:00:23 -07:00 · 2383e45f1d
commit 2383e45f1d
parent 348f6117c1 f616549124
7 changed files with 251 additions and 14 deletions
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@ -1133,12 +1133,14 @@ static int add_exception_handler(const struct bpf_insn *insn,
 		return 0;

 	if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
-		BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
-			BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
-				BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
+	    BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
+	    BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
+	    BPF_MODE(insn->code) != BPF_PROBE_MEM32SX &&
+	    BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
 		return 0;

 	is_arena = (BPF_MODE(insn->code) == BPF_PROBE_MEM32) ||
+		   (BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) ||
 		   (BPF_MODE(insn->code) == BPF_PROBE_ATOMIC);

 	if (!ctx->prog->aux->extable ||
@ -1659,7 +1661,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 	case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
 	case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
 	case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
-		if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
+	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B:
+	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H:
+	case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W:
+		if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
+		    BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) {
 			emit(A64_ADD(1, tmp2, src, arena_vm_base), ctx);
 			src = tmp2;
 		}
@ -1671,7 +1677,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 			off_adj = off;
 		}
 		sign_extend = (BPF_MODE(insn->code) == BPF_MEMSX ||
-				BPF_MODE(insn->code) == BPF_PROBE_MEMSX);
+				BPF_MODE(insn->code) == BPF_PROBE_MEMSX ||
+				 BPF_MODE(insn->code) == BPF_PROBE_MEM32SX);
 		switch (BPF_SIZE(code)) {
 		case BPF_W:
 			if (is_lsi_offset(off_adj, 2)) {
@ -1879,9 +1886,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
 		if (ret)
 			return ret;

-		ret = add_exception_handler(insn, ctx, dst);
-		if (ret)
-			return ret;
+		if (BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
+			ret = add_exception_handler(insn, ctx, dst);
+			if (ret)
+				return ret;
+		}
 		break;

 	default:
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@ -2066,6 +2066,11 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
 		case BPF_STX | BPF_ATOMIC | BPF_DW:
 			if (insn->imm == BPF_CMPXCHG)
 				return rv_ext_enabled(ZACAS);
+			break;
+		case BPF_LDX | BPF_MEMSX | BPF_B:
+		case BPF_LDX | BPF_MEMSX | BPF_H:
+		case BPF_LDX | BPF_MEMSX | BPF_W:
+			return false;
 		}
 	}

--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@ -2967,6 +2967,11 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
 	case BPF_STX | BPF_ATOMIC | BPF_DW:
 		if (bpf_atomic_is_load_store(insn))
 			return false;
+		break;
+	case BPF_LDX | BPF_MEMSX | BPF_B:
+	case BPF_LDX | BPF_MEMSX | BPF_H:
+	case BPF_LDX | BPF_MEMSX | BPF_W:
+		return false;
 	}
 	return true;
 }
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@ -1152,11 +1152,38 @@ static void emit_ldx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 i
 	*pprog = prog;
 }

+static void emit_ldsx_index(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, u32 index_reg, int off)
+{
+	u8 *prog = *pprog;
+
+	switch (size) {
+	case BPF_B:
+		/* movsx rax, byte ptr [rax + r12 + off] */
+		EMIT3(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x0F, 0xBE);
+		break;
+	case BPF_H:
+		/* movsx rax, word ptr [rax + r12 + off] */
+		EMIT3(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x0F, 0xBF);
+		break;
+	case BPF_W:
+		/* movsx rax, dword ptr [rax + r12 + off] */
+		EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x63);
+		break;
+	}
+	emit_insn_suffix_SIB(&prog, src_reg, dst_reg, index_reg, off);
+	*pprog = prog;
+}
+
 static void emit_ldx_r12(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
 {
 	emit_ldx_index(pprog, size, dst_reg, src_reg, X86_REG_R12, off);
 }

+static void emit_ldsx_r12(u8 **prog, u32 size, u32 dst_reg, u32 src_reg, int off)
+{
+	emit_ldsx_index(prog, size, dst_reg, src_reg, X86_REG_R12, off);
+}
+
 /* STX: *(u8*)(dst_reg + off) = src_reg */
 static void emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off)
 {
@ -2109,15 +2136,22 @@ st:			if (is_imm8(insn->off))
 		case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
 		case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
 		case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
+		case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B:
+		case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H:
+		case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W:
 		case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
 		case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
 		case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
 		case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
 			start_of_ldx = prog;
-			if (BPF_CLASS(insn->code) == BPF_LDX)
-				emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
-			else
+			if (BPF_CLASS(insn->code) == BPF_LDX) {
+				if (BPF_MODE(insn->code) == BPF_PROBE_MEM32SX)
+					emit_ldsx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
+				else
+					emit_ldx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
+			} else {
 				emit_stx_r12(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
+			}
 populate_extable:
 			{
 				struct exception_table_entry *ex;
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@ -78,6 +78,9 @@ struct ctl_table_header;
 /* unused opcode to mark special atomic instruction */
 #define BPF_PROBE_ATOMIC 0xe0

+/* unused opcode to mark special ldsx instruction. Same as BPF_NOSPEC */
+#define BPF_PROBE_MEM32SX 0xc0
+
 /* unused opcode to mark call to interpreter with arguments */
 #define BPF_CALL_ARGS	0xe0

--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@ -21379,10 +21379,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 			continue;
 		case PTR_TO_ARENA:
 			if (BPF_MODE(insn->code) == BPF_MEMSX) {
-				verbose(env, "sign extending loads from arena are not supported yet\n");
-				return -EOPNOTSUPP;
+				if (!bpf_jit_supports_insn(insn, true)) {
+					verbose(env, "sign extending loads from arena are not supported yet\n");
+					return -EOPNOTSUPP;
+				}
+				insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32SX | BPF_SIZE(insn->code);
+			} else {
+				insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
 			}
-			insn->code = BPF_CLASS(insn->code) | BPF_PROBE_MEM32 | BPF_SIZE(insn->code);
 			env->prog->aux->num_exentries++;
 			continue;
 		default:
@ -21588,6 +21592,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 			if (BPF_CLASS(insn->code) == BPF_LDX &&
 			    (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
 			     BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
+			     BPF_MODE(insn->code) == BPF_PROBE_MEM32SX ||
 			     BPF_MODE(insn->code) == BPF_PROBE_MEMSX))
 				num_exentries++;
 			if ((BPF_CLASS(insn->code) == BPF_STX ||
--- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c
+++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c
@ -3,6 +3,7 @@
 #include <linux/bpf.h>
 #include <bpf/bpf_helpers.h>
 #include "bpf_misc.h"
+#include "bpf_arena_common.h"

 #if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
 	(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
@ -10,6 +11,12 @@
 	defined(__TARGET_ARCH_loongarch)) && \
 	__clang_major__ >= 18

+struct {
+	__uint(type, BPF_MAP_TYPE_ARENA);
+	__uint(map_flags, BPF_F_MMAPABLE);
+	__uint(max_entries, 1);
+} arena SEC(".maps");
+
 SEC("socket")
 __description("LDSX, S8")
 __success __success_unpriv __retval(-2)
@ -256,6 +263,175 @@ __naked void ldsx_ctx_8(void)
 	: __clobber_all);
 }

+SEC("syscall")
+__description("Arena LDSX Disasm")
+__success
+__arch_x86_64
+__jited("movslq	0x10(%rax,%r12), %r14")
+__jited("movswq	0x18(%rax,%r12), %r14")
+__jited("movsbq	0x20(%rax,%r12), %r14")
+__jited("movslq	0x10(%rdi,%r12), %r15")
+__jited("movswq	0x18(%rdi,%r12), %r15")
+__jited("movsbq	0x20(%rdi,%r12), %r15")
+__arch_arm64
+__jited("add	x11, x7, x28")
+__jited("ldrsw	x21, [x11, #0x10]")
+__jited("add	x11, x7, x28")
+__jited("ldrsh	x21, [x11, #0x18]")
+__jited("add	x11, x7, x28")
+__jited("ldrsb	x21, [x11, #0x20]")
+__jited("add	x11, x0, x28")
+__jited("ldrsw	x22, [x11, #0x10]")
+__jited("add	x11, x0, x28")
+__jited("ldrsh	x22, [x11, #0x18]")
+__jited("add	x11, x0, x28")
+__jited("ldrsb	x22, [x11, #0x20]")
+__naked void arena_ldsx_disasm(void *ctx)
+{
+	asm volatile (
+	"r1 = %[arena] ll;"
+	"r2 = 0;"
+	"r3 = 1;"
+	"r4 = %[numa_no_node];"
+	"r5 = 0;"
+	"call %[bpf_arena_alloc_pages];"
+	"r0 = addr_space_cast(r0, 0x0, 0x1);"
+	"r1 = r0;"
+	"r8 = *(s32 *)(r0 + 16);"
+	"r8 = *(s16 *)(r0 + 24);"
+	"r8 = *(s8  *)(r0 + 32);"
+	"r9 = *(s32 *)(r1 + 16);"
+	"r9 = *(s16 *)(r1 + 24);"
+	"r9 = *(s8  *)(r1 + 32);"
+	"r0 = 0;"
+	"exit;"
+	:: __imm(bpf_arena_alloc_pages),
+	   __imm_addr(arena),
+	   __imm_const(numa_no_node, NUMA_NO_NODE)
+	:  __clobber_all
+	);
+}
+
+SEC("syscall")
+__description("Arena LDSX Exception")
+__success __retval(0)
+__arch_x86_64
+__arch_arm64
+__naked void arena_ldsx_exception(void *ctx)
+{
+	asm volatile (
+	"r1 = %[arena] ll;"
+	"r0 = 0xdeadbeef;"
+	"r0 = addr_space_cast(r0, 0x0, 0x1);"
+	"r1 = 0x3fe;"
+	"*(u64 *)(r0 + 0) = r1;"
+	"r0 = *(s8 *)(r0 + 0);"
+	"exit;"
+	:
+	:  __imm_addr(arena)
+	:  __clobber_all
+	);
+}
+
+SEC("syscall")
+__description("Arena LDSX, S8")
+__success __retval(-1)
+__arch_x86_64
+__arch_arm64
+__naked void arena_ldsx_s8(void *ctx)
+{
+	asm volatile (
+	"r1 = %[arena] ll;"
+	"r2 = 0;"
+	"r3 = 1;"
+	"r4 = %[numa_no_node];"
+	"r5 = 0;"
+	"call %[bpf_arena_alloc_pages];"
+	"r0 = addr_space_cast(r0, 0x0, 0x1);"
+	"r1 = 0x3fe;"
+	"*(u64 *)(r0 + 0) = r1;"
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	"r0 = *(s8 *)(r0 + 0);"
+#else
+	"r0 = *(s8 *)(r0 + 7);"
+#endif
+	"r0 >>= 1;"
+	"exit;"
+	:: __imm(bpf_arena_alloc_pages),
+	   __imm_addr(arena),
+	   __imm_const(numa_no_node, NUMA_NO_NODE)
+	:  __clobber_all
+	);
+}
+
+SEC("syscall")
+__description("Arena LDSX, S16")
+__success __retval(-1)
+__arch_x86_64
+__arch_arm64
+__naked void arena_ldsx_s16(void *ctx)
+{
+	asm volatile (
+	"r1 = %[arena] ll;"
+	"r2 = 0;"
+	"r3 = 1;"
+	"r4 = %[numa_no_node];"
+	"r5 = 0;"
+	"call %[bpf_arena_alloc_pages];"
+	"r0 = addr_space_cast(r0, 0x0, 0x1);"
+	"r1 = 0x3fffe;"
+	"*(u64 *)(r0 + 0) = r1;"
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	"r0 = *(s16 *)(r0 + 0);"
+#else
+	"r0 = *(s16 *)(r0 + 6);"
+#endif
+	"r0 >>= 1;"
+	"exit;"
+	:: __imm(bpf_arena_alloc_pages),
+	   __imm_addr(arena),
+	   __imm_const(numa_no_node, NUMA_NO_NODE)
+	:  __clobber_all
+	);
+}
+
+SEC("syscall")
+__description("Arena LDSX, S32")
+__success __retval(-1)
+__arch_x86_64
+__arch_arm64
+__naked void arena_ldsx_s32(void *ctx)
+{
+	asm volatile (
+	"r1 = %[arena] ll;"
+	"r2 = 0;"
+	"r3 = 1;"
+	"r4 = %[numa_no_node];"
+	"r5 = 0;"
+	"call %[bpf_arena_alloc_pages];"
+	"r0 = addr_space_cast(r0, 0x0, 0x1);"
+	"r1 = 0xfffffffe;"
+	"*(u64 *)(r0 + 0) = r1;"
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	"r0 = *(s32 *)(r0 + 0);"
+#else
+	"r0 = *(s32 *)(r0 + 4);"
+#endif
+	"r0 >>= 1;"
+	"exit;"
+	:: __imm(bpf_arena_alloc_pages),
+	   __imm_addr(arena),
+	   __imm_const(numa_no_node, NUMA_NO_NODE)
+	:  __clobber_all
+	);
+}
+
+/* to retain debug info for BTF generation */
+void kfunc_root(void)
+{
+	bpf_arena_alloc_pages(0, 0, 0, 0, 0);
+}
+
 #else

 SEC("socket")