github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf, x86: Compare the PROBE_MEM address range with a shift

Browse Source 
The PROBE_MEM check rejects src_reg + off - VSYSCALL_ADDR up to a limit
of TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR, which does not fit into a
32-bit immediate and therefore needed a movabs into r10 and a register
compare. Compare against the next power of two above the limit instead,
which is a shift by fls64(limit) followed by a jne on the result. The
addresses between the limit and that power of two are non-canonical on
every x86-64 layout, since user space ends at TASK_SIZE_MAX and the
kernel half starts far above 2^57, so they faulted and had the register
zeroed by the exception handler before. Now they are zeroed by the check
without taking the fault. The compare and branch stay two uops on Intel
and AMD, while the sequence shrinks from 15 to 6 bytes.

Before, for the compare of a 4-level paging kernel:

  49 ba 00 00 a0 00 00 80 00 00  movabsq $0x800000a00000,%r10
  4d 39 d3                       cmpq %r10,%r11
  77 05                          ja 0x...

After:

  49 c1 eb 30                    shrq $0x30,%r11
  75 05                          jne 0x...

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20261013171230.523677-13-daniel@iogearbox.net
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
This commit is contained in:
Daniel Borkmann 2026-10-13 19:12:24 +02:00 committed by Kumar Kartikeya Dwivedi
parent 74309a9294
commit 181576ebd8
No known key found for this signature in database
GPG Key ID: 472D377B63542F83
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
arch/x86/net/bpf_jit_comp.c
View File

@ -2848,6 +2848,11 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
* src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE
* and
* src_reg + insn->off < VSYSCALL_ADDR
*
* The check is done on src_reg + insn->off - VSYSCALL_ADDR
* against the next power of two above the limit. Addresses
* between the limit and that power of two are non-canonical
* and would fault anyway, so they get zeroed a bit earlier.
*/
u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR;
@ -2864,16 +2869,11 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
emit_insn_suffix(&prog, src_reg, AUX_REG,
insn->off - (s32)VSYSCALL_ADDR);
/* movabsq r10, limit */
emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32,
(u32)(long)limit);
/* shr r11, fls64(limit) */
emit_shift_imm(&prog, BPF_RSH, AUX_REG, fls64(limit), true);
/* cmp r10, r11 */
maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true);
EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX));
/* if unsigned '>', goto load */
EMIT2(X86_JA, 0);
/* if not zero, goto load */
EMIT2(X86_JNE, 0);
end_of_jmp = prog;
/* xor dst_reg, dst_reg */
@ -2881,7 +2881,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
/* jmp byte_after_ldx */
EMIT2(0xEB, 0);
/* populate jmp_offset for JAE above to jump to start_of_ldx */
/* populate jmp_offset for JNE above to jump to start_of_ldx */
start_of_ldx = prog;
end_of_jmp[-1] = start_of_ldx - end_of_jmp;
} else if (!accesses_stack_only) {