github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf, x86: Use shorter and faster forms for some ALU immediates

Browse Source 
An add or sub of 1 or -1 is an inc or dec, which is a byte shorter and a
single uop on all current cores, and the flags they leave untouched are
not read across BPF instructions anyway. An add of 128 is the sub of
-128 and the other way round, which fits into an imm8. An and of a 64-bit
register with a non-negative immediate clears the upper half just like
the 32-bit and does, which drops the REX.W prefix on the lower eight
registers. A shift left by one is the add of the register to itself, the
same size and a single uop that runs on more ports. LLVM emits all of
them, r += 1 for every counter, r &= mask for the masks beyond the
byte and word ones and r <<= 1 for x * 2 and x + x, and does the same
replacements for native code.

Before, for r1 += 1, r1 += 128, r1 &= 0x3ff and r1 <<= 1:

  48 83 c7 01           addq $0x1,%rdi
  48 81 c7 80 00 00 00  addq $0x80,%rdi
  48 81 e7 ff 03 00 00  andq $0x3ff,%rdi
  48 d1 e7              shlq %rdi

After:

  48 ff c7              incq %rdi
  48 83 ef 80           subq $-0x80,%rdi
  81 e7 ff 03 00 00     andl $0x3ff,%edi
  48 01 ff              addq %rdi,%rdi

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20261013171230.523677-9-daniel@iogearbox.net
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
This commit is contained in:
Daniel Borkmann 2026-10-13 19:12:20 +02:00 committed by Kumar Kartikeya Dwivedi
parent 707e1331f5
commit d59f47e031
No known key found for this signature in database
GPG Key ID: 472D377B63542F83
1 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
arch/x86/net/bpf_jit_comp.c
View File

@ -1219,9 +1219,22 @@ static void emit_alu_imm(u8 **pprog, u32 op, u32 dst_reg, s32 imm, bool is64)
[BPF_OR] = 0x0D, [BPF_XOR] = 0x35,
};
u8 *prog = *pprog;
bool inc;
/* A non-negative mask clears the upper half like the 32-bit and does */
if (op == BPF_AND && imm >= 0)
is64 = false;
/* add 128 is sub -128 and the other way round, which fit into an imm8 */
if ((op == BPF_ADD || op == BPF_SUB) && imm == 128) {
op = op == BPF_ADD ? BPF_SUB : BPF_ADD;
imm = -128;
}
inc = op == BPF_ADD ? imm == 1 : imm == -1;
maybe_emit_1mod(&prog, dst_reg, is64);
if (is_imm8(imm))
/* inc or dec for +-1, the flags are not read across instructions */
if ((op == BPF_ADD || op == BPF_SUB) && (imm == 1 || imm == -1))
EMIT2(0xFF, add_1reg(inc ? 0xC0 : 0xC8, dst_reg));
else if (is_imm8(imm))
EMIT3(0x83, add_1reg(modrm[op], dst_reg), imm);
else if (is_axreg(dst_reg))
EMIT1_off32(short_op[op], imm);
@ -1247,11 +1260,16 @@ static void emit_shift_imm(u8 **pprog, u32 op, u32 reg, u8 imm, bool is64)
{
u8 *prog = *pprog;
maybe_emit_1mod(&prog, reg, is64);
if (imm == 1)
EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg));
else
EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm);
/* The shift left by one is an add of the register to itself */
if (op == BPF_LSH && imm == 1) {
emit_alu_reg(&prog, BPF_ADD, reg, reg, is64);
} else {
maybe_emit_1mod(&prog, reg, is64);
if (imm == 1)
EMIT2(0xD1, add_1reg(simple_alu_opcodes[op], reg));
else
EMIT3(0xC1, add_1reg(simple_alu_opcodes[op], reg), imm);
}
*pprog = prog;
}