github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-27 08:33:17 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

powerpc64/bpf: Introduce bpf_jit_emit_atomic_ops() to emit atomic instructions

Browse Source 
The existing code for emitting bpf atomic instruction sequences for
atomic operations such as XCHG, CMPXCHG, ADD, AND, OR, and XOR has been
refactored into a reusable function, bpf_jit_emit_ppc_atomic_op().
It also computes the jump offset and tracks the instruction index for jited
LDARX/LWARX to be used in case it causes a fault.

Reviewed-by: Hari Bathini <hbathini@linux.ibm.com>
Tested-by: Venkat Rao Bagalkote <venkat88@linux.ibm.com>
Signed-off-by: Saket Kumar Bhaskar <skb99@linux.ibm.com>
Signed-off-by: Madhavan Srinivasan <maddy@linux.ibm.com>
Link: https://patch.msgid.link/20250904100835.1100423-4-skb99@linux.ibm.com
This commit is contained in:
Saket Kumar Bhaskar 2025-09-04 15:38:34 +05:30 committed by Madhavan Srinivasan
parent a2485d06ca
commit 45ed2e8b05
1 changed files with 115 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
arch/powerpc/net/bpf_jit_comp64.c
View File

@ -423,6 +423,111 @@ asm (
" blr ;"
);
static int bpf_jit_emit_atomic_ops(u32 *image, struct codegen_context *ctx,
const struct bpf_insn *insn, u32 *jmp_off,
u32 *tmp_idx, u32 *addrp)
{
u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
u32 size = BPF_SIZE(insn->code);
u32 src_reg = bpf_to_ppc(insn->src_reg);
u32 dst_reg = bpf_to_ppc(insn->dst_reg);
s32 imm = insn->imm;
u32 save_reg = tmp2_reg;
u32 ret_reg = src_reg;
u32 fixup_idx;
/* Get offset into TMP_REG_1 */
EMIT(PPC_RAW_LI(tmp1_reg, insn->off));
/*
* Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'
* before and after the operation.
*
* This is a requirement in the Linux Kernel Memory Model.
* See __cmpxchg_u64() in asm/cmpxchg.h as an example.
*/
if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
EMIT(PPC_RAW_SYNC());
*tmp_idx = ctx->idx;
/* load value from memory into TMP_REG_2 */
if (size == BPF_DW)
EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
else
EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
/* Save old value in _R0 */
if (imm & BPF_FETCH)
EMIT(PPC_RAW_MR(_R0, tmp2_reg));
switch (imm) {
case BPF_ADD:
case BPF_ADD | BPF_FETCH:
EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
break;
case BPF_AND:
case BPF_AND | BPF_FETCH:
EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
break;
case BPF_OR:
case BPF_OR | BPF_FETCH:
EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
break;
case BPF_XOR:
case BPF_XOR | BPF_FETCH:
EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
break;
case BPF_CMPXCHG:
/*
* Return old value in BPF_REG_0 for BPF_CMPXCHG &
* in src_reg for other cases.
*/
ret_reg = bpf_to_ppc(BPF_REG_0);
/* Compare with old value in BPF_R0 */
if (size == BPF_DW)
EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
else
EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
/* Don't set if different from old value */
PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
fallthrough;
case BPF_XCHG:
save_reg = src_reg;
break;
default:
return -EOPNOTSUPP;
}
/* store new value */
if (size == BPF_DW)
EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
else
EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
/* we're done if this succeeded */
PPC_BCC_SHORT(COND_NE, *tmp_idx * 4);
fixup_idx = ctx->idx;
if (imm & BPF_FETCH) {
/* Emit 'sync' to enforce full ordering */
if (IS_ENABLED(CONFIG_SMP))
EMIT(PPC_RAW_SYNC());
EMIT(PPC_RAW_MR(ret_reg, _R0));
/*
* Skip unnecessary zero-extension for 32-bit cmpxchg.
* For context, see commit 39491867ace5.
*/
if (size != BPF_DW && imm == BPF_CMPXCHG &&
insn_is_zext(insn + 1))
*addrp = ctx->idx * 4;
}
*jmp_off = (fixup_idx - *tmp_idx) * 4;
return 0;
}
static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off,
u32 code, u32 *image)
{
@ -538,7 +643,6 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
u32 size = BPF_SIZE(code);
u32 tmp1_reg = bpf_to_ppc(TMP_REG_1);
u32 tmp2_reg = bpf_to_ppc(TMP_REG_2);
u32 save_reg, ret_reg;
s16 off = insn[i].off;
s32 imm = insn[i].imm;
bool func_addr_fixed;
@ -546,6 +650,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
u64 imm64;
u32 true_cond;
u32 tmp_idx;
u32 jmp_off;
/*
* addrs[] maps a BPF bytecode address into a real offset from
@ -1080,93 +1185,15 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
return -EOPNOTSUPP;
}
save_reg = tmp2_reg;
ret_reg = src_reg;
/* Get offset into TMP_REG_1 */
EMIT(PPC_RAW_LI(tmp1_reg, off));
/*
* Enforce full ordering for operations with BPF_FETCH by emitting a 'sync'
* before and after the operation.
*
* This is a requirement in the Linux Kernel Memory Model.
* See __cmpxchg_u64() in asm/cmpxchg.h as an example.
*/
if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP))
EMIT(PPC_RAW_SYNC());
tmp_idx = ctx->idx * 4;
/* load value from memory into TMP_REG_2 */
if (size == BPF_DW)
EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0));
else
EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0));
/* Save old value in _R0 */
if (imm & BPF_FETCH)
EMIT(PPC_RAW_MR(_R0, tmp2_reg));
switch (imm) {
case BPF_ADD:
case BPF_ADD | BPF_FETCH:
EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
break;
case BPF_AND:
case BPF_AND | BPF_FETCH:
EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg));
break;
case BPF_OR:
case BPF_OR | BPF_FETCH:
EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg));
break;
case BPF_XOR:
case BPF_XOR | BPF_FETCH:
EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg));
break;
case BPF_CMPXCHG:
/*
* Return old value in BPF_REG_0 for BPF_CMPXCHG &
* in src_reg for other cases.
*/
ret_reg = bpf_to_ppc(BPF_REG_0);
/* Compare with old value in BPF_R0 */
if (size == BPF_DW)
EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg));
else
EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg));
/* Don't set if different from old value */
PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4);
fallthrough;
case BPF_XCHG:
save_reg = src_reg;
break;
default:
pr_err_ratelimited(
"eBPF filter atomic op code %02x (@%d) unsupported\n",
code, i);
return -EOPNOTSUPP;
}
/* store new value */
if (size == BPF_DW)
EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg));
else
EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg));
/* we're done if this succeeded */
PPC_BCC_SHORT(COND_NE, tmp_idx);
if (imm & BPF_FETCH) {
/* Emit 'sync' to enforce full ordering */
if (IS_ENABLED(CONFIG_SMP))
EMIT(PPC_RAW_SYNC());
EMIT(PPC_RAW_MR(ret_reg, _R0));
/*
* Skip unnecessary zero-extension for 32-bit cmpxchg.
* For context, see commit 39491867ace5.
*/
if (size != BPF_DW && imm == BPF_CMPXCHG &&
insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;
ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i],
&jmp_off, &tmp_idx, &addrs[i + 1]);
if (ret) {
if (ret == -EOPNOTSUPP) {
pr_err_ratelimited(
"eBPF filter atomic op code %02x (@%d) unsupported\n",
code, i);
}
return ret;
}
break;