From 181576ebd87d90aafeedbecf7a2dcc109dcff391 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:24 +0200 Subject: [PATCH] bpf, x86: Compare the PROBE_MEM address range with a shift The PROBE_MEM check rejects src_reg + off - VSYSCALL_ADDR up to a limit of TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR, which does not fit into a 32-bit immediate and therefore needed a movabs into r10 and a register compare. Compare against the next power of two above the limit instead, which is a shift by fls64(limit) followed by a jne on the result. The addresses between the limit and that power of two are non-canonical on every x86-64 layout, since user space ends at TASK_SIZE_MAX and the kernel half starts far above 2^57, so they faulted and had the register zeroed by the exception handler before. Now they are zeroed by the check without taking the fault. The compare and branch stay two uops on Intel and AMD, while the sequence shrinks from 15 to 6 bytes. Before, for the compare of a 4-level paging kernel: 49 ba 00 00 a0 00 00 80 00 00 movabsq $0x800000a00000,%r10 4d 39 d3 cmpq %r10,%r11 77 05 ja 0x... After: 49 c1 eb 30 shrq $0x30,%r11 75 05 jne 0x... Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-13-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 75194f9caa5a..564cead6eab7 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2848,6 +2848,11 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE * and * src_reg + insn->off < VSYSCALL_ADDR + * + * The check is done on src_reg + insn->off - VSYSCALL_ADDR + * against the next power of two above the limit. Addresses + * between the limit and that power of two are non-canonical + * and would fault anyway, so they get zeroed a bit earlier. */ u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR; @@ -2864,16 +2869,11 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * emit_insn_suffix(&prog, src_reg, AUX_REG, insn->off - (s32)VSYSCALL_ADDR); - /* movabsq r10, limit */ - emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32, - (u32)(long)limit); + /* shr r11, fls64(limit) */ + emit_shift_imm(&prog, BPF_RSH, AUX_REG, fls64(limit), true); - /* cmp r10, r11 */ - maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); - EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); - - /* if unsigned '>', goto load */ - EMIT2(X86_JA, 0); + /* if not zero, goto load */ + EMIT2(X86_JNE, 0); end_of_jmp = prog; /* xor dst_reg, dst_reg */ @@ -2881,7 +2881,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * /* jmp byte_after_ldx */ EMIT2(0xEB, 0); - /* populate jmp_offset for JAE above to jump to start_of_ldx */ + /* populate jmp_offset for JNE above to jump to start_of_ldx */ start_of_ldx = prog; end_of_jmp[-1] = start_of_ldx - end_of_jmp; } else if (!accesses_stack_only) {