github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-23 05:04:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 's390-bpf-support-load-acquire-and-store-release-instructions'

Browse Source 
Maxim Khmelevskii says:

====================
s390/bpf: Support load-acquire and store-release instructions

Support load-acquire (BPF_LOAD_ACQ) and store-release (BPF_STORE_REL)
instructions. Since s390 has strong memory model, implement
them as regular BPF_LDX/BPF_STX instructions.
====================

Link: https://patch.msgid.link/20260723140648.583055-5-max@linux.ibm.com
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
This commit is contained in:
Kumar Kartikeya Dwivedi 2026-07-24 23:03:09 +02:00
commit 1adb5b8931
No known key found for this signature in database
GPG Key ID: 472D377B63542F83
3 changed files with 123 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

198
arch/s390/net/bpf_jit_comp.c
View File

@ -743,10 +743,12 @@ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn,
{
if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
BPF_MODE(insn->code) != BPF_PROBE_MEMSX &&
BPF_MODE(insn->code) != BPF_PROBE_MEM32)
BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
return;
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) {
if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 ||
BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) {
/* lgrl %r1,kern_arena */
EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena);
probe->arena_reg = REG_W1;
@ -758,7 +760,8 @@ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn,
static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn,
struct bpf_jit_probe *probe)
{
if (BPF_MODE(insn->code) != BPF_PROBE_MEM32)
if (BPF_MODE(insn->code) != BPF_PROBE_MEM32 &&
BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)
return;
/* lgrl %r1,kern_arena */
@ -830,6 +833,72 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp,
return 0;
}
static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn)
{
struct bpf_jit_probe probe;
bpf_jit_probe_init(&probe);
bpf_jit_probe_load_pre(jit, insn, &probe);
switch (BPF_SIZE(insn->code)) {
case BPF_B: /* dst = *(u8 *)(ul) (src + off) */
/* llgc %dst,off(%src,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0090, insn->dst_reg, insn->src_reg,
probe.arena_reg, insn->off);
break;
case BPF_H: /* dst = *(u16 *)(ul) (src + off) */
/* llgh %dst,off(%src,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0091, insn->dst_reg, insn->src_reg,
probe.arena_reg, insn->off);
break;
case BPF_W: /* dst = *(u32 *)(ul) (src + off) */
/* llgf %dst,off(%src,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0016, insn->dst_reg, insn->src_reg,
probe.arena_reg, insn->off);
break;
case BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
/* lg %dst,off(%src,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0004, insn->dst_reg, insn->src_reg,
probe.arena_reg, insn->off);
break;
}
return bpf_jit_probe_post(jit, fp, &probe);
}
static int emit_stx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn)
{
struct bpf_jit_probe probe;
bpf_jit_probe_init(&probe);
bpf_jit_probe_store_pre(jit, insn, &probe);
switch (BPF_SIZE(insn->code)) {
case BPF_B: /* *(u8 *)(dst + off) = src_reg */
/* stcy %src,off(%dst,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0072, insn->src_reg, insn->dst_reg,
probe.arena_reg, insn->off);
break;
case BPF_H: /* (u16 *)(dst + off) = src */
/* sthy %src,off(%dst,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0070, insn->src_reg, insn->dst_reg,
probe.arena_reg, insn->off);
break;
case BPF_W: /* *(u32 *)(dst + off) = src */
/* sty %src,off(%dst,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0050, insn->src_reg, insn->dst_reg,
probe.arena_reg, insn->off);
break;
case BPF_DW: /* (u64 *)(dst + off) = src */
/* stg %src,off(%dst,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0024, insn->src_reg, insn->dst_reg,
probe.arena_reg, insn->off);
break;
}
return bpf_jit_probe_post(jit, fp, &probe);
}
/*
* Sign- or zero-extend the register if necessary
*/
@ -1477,44 +1546,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
*/
case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */
case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
bpf_jit_probe_store_pre(jit, insn, &probe);
/* stcy %src,off(%dst,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg,
probe.arena_reg, off);
err = bpf_jit_probe_post(jit, fp, &probe);
if (err < 0)
return err;
jit->seen |= SEEN_MEM;
break;
case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
bpf_jit_probe_store_pre(jit, insn, &probe);
/* sthy %src,off(%dst,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg,
probe.arena_reg, off);
err = bpf_jit_probe_post(jit, fp, &probe);
if (err < 0)
return err;
jit->seen |= SEEN_MEM;
break;
case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
bpf_jit_probe_store_pre(jit, insn, &probe);
/* sty %src,off(%dst,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg,
probe.arena_reg, off);
err = bpf_jit_probe_post(jit, fp, &probe);
if (err < 0)
return err;
jit->seen |= SEEN_MEM;
break;
case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
case BPF_STX | BPF_PROBE_MEM32 | BPF_DW:
bpf_jit_probe_store_pre(jit, insn, &probe);
/* stg %src,off(%dst,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg,
probe.arena_reg, off);
err = bpf_jit_probe_post(jit, fp, &probe);
err = emit_stx(jit, fp, insn);
if (err < 0)
return err;
jit->seen |= SEEN_MEM;
@ -1574,19 +1612,23 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
/*
* BPF_ATOMIC
*/
case BPF_STX | BPF_ATOMIC | BPF_B:
case BPF_STX | BPF_ATOMIC | BPF_H:
case BPF_STX | BPF_ATOMIC | BPF_DW:
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
{
bool is32 = BPF_SIZE(insn->code) == BPF_W;
/*
* Unlike loads and stores, atomics have only a base register,
* but no index register. For the non-arena case, simply use
* %dst as a base. For the arena case, use the work register
* %r1: first, load the arena base into it, and then add %dst
* to it.
* Unlike loads and stores, s390 atomics have only a base
* register, but no index register. For the non-arena case,
* simply use %dst as a base. For the arena case, use the
* work register %r1: first, load the arena base into it,
* and then add %dst to it.
*/
probe.arena_reg = dst_reg;
@ -1673,6 +1715,18 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
if (err < 0)
return err;
break;
case BPF_LOAD_ACQ:
/* s390 has strong ordering, just use load */
err = emit_ldx(jit, fp, insn);
if (err < 0)
return err;
break;
case BPF_STORE_REL:
/* s390 has strong ordering, just use store */
err = emit_stx(jit, fp, insn);
if (err < 0)
return err;
break;
default:
pr_err("Unknown atomic operation %02x\n", insn->imm);
return -1;
@ -1687,15 +1741,20 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEM | BPF_B:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_B:
bpf_jit_probe_load_pre(jit, insn, &probe);
/* llgc %dst,off(%src,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg,
probe.arena_reg, off);
err = bpf_jit_probe_post(jit, fp, &probe);
case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
err = emit_ldx(jit, fp, insn);
if (err < 0)
return err;
jit->seen |= SEEN_MEM;
if (insn_is_zext(&insn[1]))
if (BPF_SIZE(insn->code) != BPF_DW && insn_is_zext(&insn[1]))
insn_count = 2;
break;
case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */
@ -1708,20 +1767,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
return err;
jit->seen |= SEEN_MEM;
break;
case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_H:
bpf_jit_probe_load_pre(jit, insn, &probe);
/* llgh %dst,off(%src,%arena) */
EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg,
probe.arena_reg, off);
err = bpf_jit_probe_post(jit, fp, &probe);
if (err < 0)
return err;
jit->seen |= SEEN_MEM;
if (insn_is_zext(&insn[1]))
insn_count = 2;
break;
case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
bpf_jit_probe_load_pre(jit, insn, &probe);
@ -1732,20 +1777,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
return err;
jit->seen |= SEEN_MEM;
break;
case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_W:
bpf_jit_probe_load_pre(jit, insn, &probe);
/* llgf %dst,off(%src) */
jit->seen |= SEEN_MEM;
EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg,
probe.arena_reg, off);
err = bpf_jit_probe_post(jit, fp, &probe);
if (err < 0)
return err;
if (insn_is_zext(&insn[1]))
insn_count = 2;
break;
case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
bpf_jit_probe_load_pre(jit, insn, &probe);
@ -1756,18 +1787,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
if (err < 0)
return err;
break;
case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW:
bpf_jit_probe_load_pre(jit, insn, &probe);
/* lg %dst,off(%src,%arena) */
jit->seen |= SEEN_MEM;
EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg,
probe.arena_reg, off);
err = bpf_jit_probe_post(jit, fp, &probe);
if (err < 0)
return err;
break;
/*
* BPF_JMP / CALL
*/
@ -3028,13 +3047,6 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
if (!in_arena)
return true;
switch (insn->code) {
case BPF_STX | BPF_ATOMIC | BPF_B:
case BPF_STX | BPF_ATOMIC | BPF_H:
case BPF_STX | BPF_ATOMIC | BPF_W:
case BPF_STX | BPF_ATOMIC | BPF_DW:
if (bpf_atomic_is_load_store(insn))
return false;
break;
case BPF_LDX | BPF_MEMSX | BPF_B:
case BPF_LDX | BPF_MEMSX | BPF_H:
case BPF_LDX | BPF_MEMSX | BPF_W:

18
tools/testing/selftests/bpf/progs/arena_atomics.c
View File

@ -28,8 +28,10 @@ bool skip_all_tests = true;
#if defined(ENABLE_ATOMICS_TESTS) && \
defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
(defined(__TARGET_ARCH_arm64) || \
defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
defined(__TARGET_ARCH_s390))
bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false;
#else
bool skip_lacq_srel_tests = true;
@ -315,8 +317,10 @@ int load_acquire(const void *ctx)
{
#if defined(ENABLE_ATOMICS_TESTS) && \
defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
(defined(__TARGET_ARCH_arm64) || \
defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
defined(__TARGET_ARCH_s390))
#define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \
{ asm volatile ( \
@ -367,8 +371,10 @@ int store_release(const void *ctx)
{
#if defined(ENABLE_ATOMICS_TESTS) && \
defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64))
(defined(__TARGET_ARCH_arm64) || \
defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
defined(__TARGET_ARCH_s390))
#define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \
{ asm volatile ( \

9
tools/testing/selftests/bpf/progs/bpf_misc.h
View File

@ -264,9 +264,12 @@
#endif
#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
(defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
defined(__TARGET_ARCH_powerpc) || defined(__TARGET_ARCH_loongarch))
(defined(__TARGET_ARCH_arm64) || \
defined(__TARGET_ARCH_x86) || \
(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
defined(__TARGET_ARCH_s390) || \
defined(__TARGET_ARCH_powerpc) || \
defined(__TARGET_ARCH_loongarch))
#define CAN_USE_LOAD_ACQ_STORE_REL
#endif