github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-29 01:23:56 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

riscv, bpf: Optimize cmpxchg insn with Zacas support

Browse Source 
Optimize cmpxchg instruction with amocas.w and amocas.d
Zacas instructions.

Signed-off-by: Pu Lehui <pulehui@huawei.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Tested-by: Björn Töpel <bjorn@rivosinc.com>
Reviewed-by: Björn Töpel <bjorn@rivosinc.com>
Acked-by: Björn Töpel <bjorn@kernel.org>
Link: https://lore.kernel.org/bpf/20250719091730.2660197-8-pulehui@huaweicloud.com
This commit is contained in:
Pu Lehui 2025-07-19 09:17:27 +00:00 committed by Daniel Borkmann
parent de39d2c4cd
commit 1c0196b878
2 changed files with 29 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
arch/riscv/net/bpf_jit.h
View File

@ -1294,6 +1294,33 @@ static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx)
emit_mv(rd, RV_REG_T2, ctx);
}
static inline void emit_cmpxchg(u8 rd, u8 rs, u8 r0, bool is64, struct rv_jit_context *ctx)
{
int jmp_offset;
if (rv_ext_enabled(ZACAS)) {
emit(is64 ? rvzacas_amocas_d(r0, rs, rd, 1, 1) :
rvzacas_amocas_w(r0, rs, rd, 1, 1), ctx);
if (!is64)
emit_zextw(r0, r0, ctx);
return;
}
if (is64)
emit_mv(RV_REG_T2, r0, ctx);
else
emit_addiw(RV_REG_T2, r0, 0, ctx);
emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
rv_lr_w(r0, 0, rd, 0, 0), ctx);
jmp_offset = ninsns_rvoff(8);
emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) :
rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
jmp_offset = ninsns_rvoff(-6);
emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
emit_fence_rw_rw(ctx);
}
#endif /* __riscv_xlen == 64 */
void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog);

18
arch/riscv/net/bpf_jit_comp64.c
View File

@ -599,10 +599,9 @@ static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
struct rv_jit_context *ctx)
{
u8 r0, code = insn->code;
u8 code = insn->code;
s16 off = insn->off;
s32 imm = insn->imm;
int jmp_offset;
bool is64;
if (BPF_SIZE(code) != BPF_W && BPF_SIZE(code) != BPF_DW) {
@ -673,20 +672,7 @@ static int emit_atomic_rmw(u8 rd, u8 rs, const struct bpf_insn *insn,
break;
/* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */
case BPF_CMPXCHG:
r0 = bpf_to_rv_reg(BPF_REG_0, ctx);
if (is64)
emit_mv(RV_REG_T2, r0, ctx);
else
emit_addiw(RV_REG_T2, r0, 0, ctx);
emit(is64 ? rv_lr_d(r0, 0, rd, 0, 0) :
rv_lr_w(r0, 0, rd, 0, 0), ctx);
jmp_offset = ninsns_rvoff(8);
emit(rv_bne(RV_REG_T2, r0, jmp_offset >> 1), ctx);
emit(is64 ? rv_sc_d(RV_REG_T3, rs, rd, 0, 1) :
rv_sc_w(RV_REG_T3, rs, rd, 0, 1), ctx);
jmp_offset = ninsns_rvoff(-6);
emit(rv_bne(RV_REG_T3, 0, jmp_offset >> 1), ctx);
emit_fence_rw_rw(ctx);
emit_cmpxchg(rd, rs, regmap[BPF_REG_0], is64, ctx);
break;
default:
pr_err_once("bpf-jit: invalid atomic RMW opcode %02x\n", imm);