From 434308885db42e7cc929e8466ac9eb7f69a999e0 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 17 Aug 2026 22:07:23 +0800 Subject: [PATCH] LoongArch: BPF: Implement branchless conditional move for TCC The current implementation handles combined bpf2bpf and tail calls by checking at runtime whether REG_TCC holds a scalar count or a pointer address via a conditional jump. This adds branch prediction overhead in the hot path of tail call execution. To implement branchless conditional move, use an unsigned comparison (sltui) combined with mask instructions (maskeqz/masknez) to achieve branchless classification and blending of incoming scalar counts and kernel pointers in REG_TCC. This optimization refactors the inner logic of the helper function, unifies the offset decrement at the function entry, and removes all runtime branching from the prologue hot path completely. Signed-off-by: Tiezhu Yang Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/inst.h | 6 ++++ arch/loongarch/net/bpf_jit.c | 47 +++++++++---------------------- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 76b723590023..19feae166e1c 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -97,6 +97,7 @@ enum reg2i6_op { }; enum reg2i12_op { + sltui_op = 0x09, addiw_op = 0x0a, addid_op = 0x0b, lu52id_op = 0x0c, @@ -153,6 +154,8 @@ enum reg3_op { addd_op = 0x21, subw_op = 0x22, subd_op = 0x23, + maskeqz_op = 0x26, + masknez_op = 0x27, nor_op = 0x28, and_op = 0x29, or_op = 0x2a, @@ -644,6 +647,7 @@ static inline void emit_##NAME(union loongarch_instruction *insn, \ insn->reg2i12_format.rj = rj; \ } +DEF_EMIT_REG2I12_FORMAT(sltui, sltui_op) DEF_EMIT_REG2I12_FORMAT(addiw, addiw_op) DEF_EMIT_REG2I12_FORMAT(addid, addid_op) DEF_EMIT_REG2I12_FORMAT(lu52id, lu52id_op) @@ -749,6 +753,8 @@ DEF_EMIT_REG3_FORMAT(divd, divd_op) DEF_EMIT_REG3_FORMAT(modd, modd_op) DEF_EMIT_REG3_FORMAT(divdu, divdu_op) DEF_EMIT_REG3_FORMAT(moddu, moddu_op) +DEF_EMIT_REG3_FORMAT(maskeqz, maskeqz_op) +DEF_EMIT_REG3_FORMAT(masknez, masknez_op) DEF_EMIT_REG3_FORMAT(and, and_op) DEF_EMIT_REG3_FORMAT(or, or_op) DEF_EMIT_REG3_FORMAT(xor, xor_op) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 846d652b0d22..f9ba7b961b04 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -52,50 +52,29 @@ static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) const struct bpf_prog *prog = ctx->prog; const bool is_main_prog = !bpf_is_subprog(prog); + *store_offset -= sizeof(long); if (is_main_prog) { - /* - * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT - * if (REG_TCC > T3 ) - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * else - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * REG_TCC = LOONGARCH_GPR_SP + store_offset - * - * std REG_TCC -> LOONGARCH_GPR_SP + store_offset - * - * The purpose of this code is to first push the TCC into stack, - * and then push the address of TCC into stack. - * In cases where bpf2bpf and tailcall are used in combination, - * the value in REG_TCC may be a count or an address, - * these two cases need to be judged and handled separately. - */ - emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); - *store_offset -= sizeof(long); - - emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4); - - /* - * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count, - * push tcc into stack - */ + /* Save entrance TCC state (scalar count or kernel pointer) to local 'tcc' slot */ emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); - /* Push the address of TCC into the REG_TCC */ - emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset); - - emit_uncond_jmp(ctx, 2); + /* Compute the absolute pointer to the local 'tcc' slot */ + emit_insn(ctx, addid, LOONGARCH_GPR_T7, LOONGARCH_GPR_SP, *store_offset); /* - * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address, - * push tcc_ptr into stack + * Branchless classification and blending: + * Combine interleaved inputs between a scalar count (0 to 33) + * and a kernel pointer address without runtime branching. */ - emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); + emit_insn(ctx, sltui, LOONGARCH_GPR_T8, REG_TCC, MAX_TAIL_CALL_CNT + 1); + emit_insn(ctx, maskeqz, LOONGARCH_GPR_T7, LOONGARCH_GPR_T7, LOONGARCH_GPR_T8); + emit_insn(ctx, masknez, REG_TCC, REG_TCC, LOONGARCH_GPR_T8); + emit_insn(ctx, or, REG_TCC, REG_TCC, LOONGARCH_GPR_T7); } else { - *store_offset -= sizeof(long); + /* Subprograms: backup the verified TCC pointer inherited via REG_TCC */ emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); } - /* Push tcc_ptr into stack */ + /* Store the finalized TCC pointer value securely into the local 'tcc_ptr' slot */ *store_offset -= sizeof(long); emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); }