From d59f47e031aecdb6d8e03d4d4d8880613339f086 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Tue, 13 Oct 2026 19:12:20 +0200 Subject: [PATCH] bpf, x86: Use shorter and faster forms for some ALU immediates An add or sub of 1 or -1 is an inc or dec, which is a byte shorter and a single uop on all current cores, and the flags they leave untouched are not read across BPF instructions anyway. An add of 128 is the sub of -128 and the other way round, which fits into an imm8. An and of a 64-bit register with a non-negative immediate clears the upper half just like the 32-bit and does, which drops the REX.W prefix on the lower eight registers. A shift left by one is the add of the register to itself, the same size and a single uop that runs on more ports. LLVM emits all of them, r += 1 for every counter, r &= mask for the masks beyond the byte and word ones and r <<= 1 for x * 2 and x + x, and does the same replacements for native code. Before, for r1 += 1, r1 += 128, r1 &= 0x3ff and r1 <<= 1: 48 83 c7 01 addq $0x1,%rdi 48 81 c7 80 00 00 00 addq $0x80,%rdi 48 81 e7 ff 03 00 00 andq $0x3ff,%rdi 48 d1 e7 shlq %rdi After: 48 ff c7 incq %rdi 48 83 ef 80 subq $-0x80,%rdi 81 e7 ff 03 00 00 andl $0x3ff,%edi 48 01 ff addq %rdi,%rdi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20261013171230.523677-9-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- arch/x86/net/bpf_jit_comp.c | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 3e367600a0ad..1c84d2a02aba 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1219,9 +1219,22 @@ static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64) [BPF_OR] = 0x0D, [BPF_XOR] = 0x35, }; u8 *prog = *pprog; + bool inc; + /* A non-negative mask clears the upper half like the 32-bit and does */ + if (op == BPF_AND && imm >= 0) + is64 = false; + /* add 128 is sub -128 and the other way round, which fit into an imm8 */ + if ((op == BPF_ADD || op == BPF_SUB) && imm == 128) { + op = op == BPF_ADD ? BPF_SUB : BPF_ADD; + imm = -128; + } + inc = op == BPF_ADD ? imm == 1 : imm == -1; maybe_emit_1mod(&prog, dst_reg, is64); - if (is_imm8(imm)) + /* inc or dec for +-1, the flags are not read across instructions */ + if ((op == BPF_ADD || op == BPF_SUB) && (imm == 1 || imm == -1)) + EMIT2(0xFF, add_1reg(inc ? 0xC0 : 0xC8, dst_reg)); + else if (is_imm8(imm)) EMIT3(0x83, add_1reg(modrm[op], dst_reg), imm); else if (is_axreg(dst_reg)) EMIT1_off32(short_op[op], imm); @@ -1247,11 +1260,16 @@ static void emit_shift_imm(u8 **pprog, u32 op, u32 reg, u8 imm, bool is64) { u8 *prog = *pprog; - maybe_emit_1mod(&prog, reg, is64); - if (imm == 1) - EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg)); - else - EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm); + /* The shift left by one is an add of the register to itself */ + if (op == BPF_LSH && imm == 1) { + emit_alu_reg(&prog, BPF_ADD, reg, reg, is64); + } else { + maybe_emit_1mod(&prog, reg, is64); + if (imm == 1) + EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg)); + else + EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm); + } *pprog = prog; }