github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-24 06:24:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

LoongArch: BPF: Align value-returning atomics with LKMM

Browse Source 
Per the Linux Kernel Memory Model, value-returning atomic RMW operations
must provide sequentially consistent ordering (a full memory barrier).
On LoongArch, plain AMO instructions and bare ll/sc loops do not satisfy
this requirement by themselves.

Update emit_atomic_rmw() to emit barrier-carrying instructions for all
value-returning BPF atomics:

 - BPF_FETCH (ADD/AND/OR/XOR): use am*_db.{b,h,w,d}
 - BPF_XCHG: use amswap_db.{b,h,w,d}
 - BPF_CMPXCHG: emit dbar 0x700 after the ll/sc loop, matching
   __WEAK_LLSC_MB in cmpxchg.h

Add the corresponding instruction encodings and emit helpers to inst.h.
Non-value-returning RMW ops (plain BPF_ADD, BPF_AND, etc.) are left as
weakly ordered, consistent with LKMM.

Acked-by: Hengqi Chen <hengqi.chen@gmail.com>
Acked-by: Tiezhu Yang <yangtiezhu@loongson.cn>
Tested-by: Tiezhu Yang <yangtiezhu@loongson.cn>
Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
This commit is contained in:
Chenguang Zhao 2026-08-17 22:07:39 +08:00 committed by Huacai Chen
parent 9fcdba3423
commit 576412d982
2 changed files with 36 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
arch/loongarch/include/asm/inst.h
View File

@ -202,6 +202,10 @@ enum reg3_op {
amswaph_op = 0x70b9,
amaddb_op = 0x70ba,
amaddh_op = 0x70bb,
amswapdbb_op = 0x70bc,
amswapdbh_op = 0x70bd,
amadddbb_op = 0x70be,
amadddbh_op = 0x70bf,
amswapw_op = 0x70c0,
amswapd_op = 0x70c1,
amaddw_op = 0x70c2,
@ -789,6 +793,20 @@ DEF_EMIT_REG3_FORMAT(amswapb, amswapb_op)
DEF_EMIT_REG3_FORMAT(amswaph, amswaph_op)
DEF_EMIT_REG3_FORMAT(amswapw, amswapw_op)
DEF_EMIT_REG3_FORMAT(amswapd, amswapd_op)
DEF_EMIT_REG3_FORMAT(amswapdbb, amswapdbb_op)
DEF_EMIT_REG3_FORMAT(amswapdbh, amswapdbh_op)
DEF_EMIT_REG3_FORMAT(amadddbb, amadddbb_op)
DEF_EMIT_REG3_FORMAT(amadddbh, amadddbh_op)
DEF_EMIT_REG3_FORMAT(amadddbw, amadddbw_op)
DEF_EMIT_REG3_FORMAT(amadddbd, amadddbd_op)
DEF_EMIT_REG3_FORMAT(amanddbw, amanddbw_op)
DEF_EMIT_REG3_FORMAT(amanddbd, amanddbd_op)
DEF_EMIT_REG3_FORMAT(amordbw, amordbw_op)
DEF_EMIT_REG3_FORMAT(amordbd, amordbd_op)
DEF_EMIT_REG3_FORMAT(amxordbw, amxordbw_op)
DEF_EMIT_REG3_FORMAT(amxordbd, amxordbd_op)
DEF_EMIT_REG3_FORMAT(amswapdbw, amswapdbw_op)
DEF_EMIT_REG3_FORMAT(amswapdbd, amswapdbd_op)
#define DEF_EMIT_REG3SA2_FORMAT(NAME, OP) \
static inline void emit_##NAME(union loongarch_instruction *insn, \

32
arch/loongarch/net/bpf_jit.c
View File

@ -8,6 +8,9 @@
#include <asm/asm-offsets.h>
#include "bpf_jit.h"
/* DBAR hint for LL/SC completion ordering, see __WEAK_LLSC_MB */
#define DBAR_LLSC_MB 0x700
#define LOONGARCH_MAX_REG_ARGS 8
#define LOONGARCH_SAVE_RA_NINSNS 1
@ -408,7 +411,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amadd.b instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amaddb, src, t1, t3);
emit_insn(ctx, amadddbb, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_H:
@ -416,39 +419,39 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amadd.h instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amaddh, src, t1, t3);
emit_insn(ctx, amadddbh, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_W:
emit_insn(ctx, amaddw, src, t1, t3);
emit_insn(ctx, amadddbw, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_DW:
emit_insn(ctx, amaddd, src, t1, t3);
emit_insn(ctx, amadddbd, src, t1, t3);
break;
}
break;
case BPF_AND | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amandd, src, t1, t3);
emit_insn(ctx, amanddbd, src, t1, t3);
} else {
emit_insn(ctx, amandw, src, t1, t3);
emit_insn(ctx, amanddbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
case BPF_OR | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amord, src, t1, t3);
emit_insn(ctx, amordbd, src, t1, t3);
} else {
emit_insn(ctx, amorw, src, t1, t3);
emit_insn(ctx, amordbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
case BPF_XOR | BPF_FETCH:
if (isdw) {
emit_insn(ctx, amxord, src, t1, t3);
emit_insn(ctx, amxordbd, src, t1, t3);
} else {
emit_insn(ctx, amxorw, src, t1, t3);
emit_insn(ctx, amxordbw, src, t1, t3);
emit_zext_32(ctx, src, true);
}
break;
@ -460,7 +463,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amswap.b instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amswapb, src, t1, t3);
emit_insn(ctx, amswapdbb, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_H:
@ -468,15 +471,15 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
pr_err_once("bpf-jit: amswap.h instruction is not supported\n");
return -EINVAL;
}
emit_insn(ctx, amswaph, src, t1, t3);
emit_insn(ctx, amswapdbh, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_W:
emit_insn(ctx, amswapw, src, t1, t3);
emit_insn(ctx, amswapdbw, src, t1, t3);
emit_zext_32(ctx, src, true);
break;
case BPF_DW:
emit_insn(ctx, amswapd, src, t1, t3);
emit_insn(ctx, amswapdbd, src, t1, t3);
break;
}
break;
@ -499,6 +502,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
emit_zext_32(ctx, r0, true);
}
emit_insn(ctx, dbar, DBAR_LLSC_MB);
break;
default:
pr_err_once("bpf-jit: invalid atomic read-modify-write opcode %02x\n", imm);