github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-24 06:24:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

LoongArch: BPF: Implement branchless conditional move for TCC

Browse Source 
The current implementation handles combined bpf2bpf and tail calls by
checking at runtime whether REG_TCC holds a scalar count or a pointer
address via a conditional jump. This adds branch prediction overhead
in the hot path of tail call execution.

To implement branchless conditional move, use an unsigned comparison
(sltui) combined with mask instructions (maskeqz/masknez) to achieve
branchless classification and blending of incoming scalar counts and
kernel pointers in REG_TCC.

This optimization refactors the inner logic of the helper function,
unifies the offset decrement at the function entry, and removes all
runtime branching from the prologue hot path completely.

Signed-off-by: Tiezhu Yang <yangtiezhu@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
This commit is contained in:
Tiezhu Yang 2026-08-17 22:07:23 +08:00 committed by Huacai Chen
parent 37d545d12f
commit 434308885d
2 changed files with 19 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
arch/loongarch/include/asm/inst.h
View File

@ -97,6 +97,7 @@ enum reg2i6_op {
};
enum reg2i12_op {
sltui_op = 0x09,
addiw_op = 0x0a,
addid_op = 0x0b,
lu52id_op = 0x0c,
@ -153,6 +154,8 @@ enum reg3_op {
addd_op = 0x21,
subw_op = 0x22,
subd_op = 0x23,
maskeqz_op = 0x26,
masknez_op = 0x27,
nor_op = 0x28,
and_op = 0x29,
or_op = 0x2a,
@ -644,6 +647,7 @@ static inline void emit_##NAME(union loongarch_instruction *insn, \
insn->reg2i12_format.rj = rj; \
}
DEF_EMIT_REG2I12_FORMAT(sltui, sltui_op)
DEF_EMIT_REG2I12_FORMAT(addiw, addiw_op)
DEF_EMIT_REG2I12_FORMAT(addid, addid_op)
DEF_EMIT_REG2I12_FORMAT(lu52id, lu52id_op)
@ -749,6 +753,8 @@ DEF_EMIT_REG3_FORMAT(divd, divd_op)
DEF_EMIT_REG3_FORMAT(modd, modd_op)
DEF_EMIT_REG3_FORMAT(divdu, divdu_op)
DEF_EMIT_REG3_FORMAT(moddu, moddu_op)
DEF_EMIT_REG3_FORMAT(maskeqz, maskeqz_op)
DEF_EMIT_REG3_FORMAT(masknez, masknez_op)
DEF_EMIT_REG3_FORMAT(and, and_op)
DEF_EMIT_REG3_FORMAT(or, or_op)
DEF_EMIT_REG3_FORMAT(xor, xor_op)

47
arch/loongarch/net/bpf_jit.c
View File

@ -52,50 +52,29 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset)
const struct bpf_prog *prog = ctx->prog;
const bool is_main_prog = !bpf_is_subprog(prog);
*store_offset -= sizeof(long);
if (is_main_prog) {
/*
* LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT
* if (REG_TCC > T3 )
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
* else
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
* REG_TCC = LOONGARCH_GPR_SP + store_offset
*
* std REG_TCC -> LOONGARCH_GPR_SP + store_offset
*
* The purpose of this code is to first push the TCC into stack,
* and then push the address of TCC into stack.
* In cases where bpf2bpf and tailcall are used in combination,
* the value in REG_TCC may be a count or an address,
* these two cases need to be judged and handled separately.
*/
emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
*store_offset -= sizeof(long);
emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4);
/*
* If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count,
* push tcc into stack
*/
/* Save entrance TCC state (scalar count or kernel pointer) to local 'tcc' slot */
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
/* Push the address of TCC into the REG_TCC */
emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
emit_uncond_jmp(ctx, 2);
/* Compute the absolute pointer to the local 'tcc' slot */
emit_insn(ctx, addid, LOONGARCH_GPR_T7, LOONGARCH_GPR_SP, *store_offset);
/*
* If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address,
* push tcc_ptr into stack
* Branchless classification and blending:
* Combine interleaved inputs between a scalar count (0 to 33)
* and a kernel pointer address without runtime branching.
*/
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
emit_insn(ctx, sltui, LOONGARCH_GPR_T8, REG_TCC, MAX_TAIL_CALL_CNT + 1);
emit_insn(ctx, maskeqz, LOONGARCH_GPR_T7, LOONGARCH_GPR_T7, LOONGARCH_GPR_T8);
emit_insn(ctx, masknez, REG_TCC, REG_TCC, LOONGARCH_GPR_T8);
emit_insn(ctx, or, REG_TCC, REG_TCC, LOONGARCH_GPR_T7);
} else {
*store_offset -= sizeof(long);
/* Subprograms: backup the verified TCC pointer inherited via REG_TCC */
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
}
/* Push tcc_ptr into stack */
/* Store the finalized TCC pointer value securely into the local 'tcc_ptr' slot */
*store_offset -= sizeof(long);
emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset);
}