diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 9ddd89f71f28..b60877478b45 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -743,10 +743,12 @@ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, { if (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && - BPF_MODE(insn->code) != BPF_PROBE_MEM32) + BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; - if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) { /* lgrl %r1,kern_arena */ EMIT6_PCREL_RILB(0xc4080000, REG_W1, jit->kern_arena); probe->arena_reg = REG_W1; @@ -758,7 +760,8 @@ static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn, static void bpf_jit_probe_store_pre(struct bpf_jit *jit, struct bpf_insn *insn, struct bpf_jit_probe *probe) { - if (BPF_MODE(insn->code) != BPF_PROBE_MEM32) + if (BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return; /* lgrl %r1,kern_arena */ @@ -830,6 +833,72 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, return 0; } +static int emit_ldx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) +{ + struct bpf_jit_probe probe; + + bpf_jit_probe_init(&probe); + bpf_jit_probe_load_pre(jit, insn, &probe); + + switch (BPF_SIZE(insn->code)) { + case BPF_B: /* dst = *(u8 *)(ul) (src + off) */ + /* llgc %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0090, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_H: /* dst = *(u16 *)(ul) (src + off) */ + /* llgh %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0091, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_W: /* dst = *(u32 *)(ul) (src + off) */ + /* llgf %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0016, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + case BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ + /* lg %dst,off(%src,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, insn->dst_reg, insn->src_reg, + probe.arena_reg, insn->off); + break; + } + + return bpf_jit_probe_post(jit, fp, &probe); +} + +static int emit_stx(struct bpf_jit *jit, struct bpf_prog *fp, struct bpf_insn *insn) +{ + struct bpf_jit_probe probe; + + bpf_jit_probe_init(&probe); + bpf_jit_probe_store_pre(jit, insn, &probe); + + switch (BPF_SIZE(insn->code)) { + case BPF_B: /* *(u8 *)(dst + off) = src_reg */ + /* stcy %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0072, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_H: /* (u16 *)(dst + off) = src */ + /* sthy %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0070, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_W: /* *(u32 *)(dst + off) = src */ + /* sty %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0050, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + case BPF_DW: /* (u64 *)(dst + off) = src */ + /* stg %src,off(%dst,%arena) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, insn->src_reg, insn->dst_reg, + probe.arena_reg, insn->off); + break; + } + + return bpf_jit_probe_post(jit, fp, &probe); +} + /* * Sign- or zero-extend the register if necessary */ @@ -1477,44 +1546,13 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, */ case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ case BPF_STX | BPF_PROBE_MEM32 | BPF_B: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* stcy %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_H: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* sthy %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_W: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* sty %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - break; case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: - bpf_jit_probe_store_pre(jit, insn, &probe); - /* stg %src,off(%dst,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); + err = emit_stx(jit, fp, insn); if (err < 0) return err; jit->seen |= SEEN_MEM; @@ -1574,19 +1612,23 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, /* * BPF_ATOMIC */ + case BPF_STX | BPF_ATOMIC | BPF_B: + case BPF_STX | BPF_ATOMIC | BPF_H: case BPF_STX | BPF_ATOMIC | BPF_DW: case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: { bool is32 = BPF_SIZE(insn->code) == BPF_W; /* - * Unlike loads and stores, atomics have only a base register, - * but no index register. For the non-arena case, simply use - * %dst as a base. For the arena case, use the work register - * %r1: first, load the arena base into it, and then add %dst - * to it. + * Unlike loads and stores, s390 atomics have only a base + * register, but no index register. For the non-arena case, + * simply use %dst as a base. For the arena case, use the + * work register %r1: first, load the arena base into it, + * and then add %dst to it. */ probe.arena_reg = dst_reg; @@ -1673,6 +1715,18 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (err < 0) return err; break; + case BPF_LOAD_ACQ: + /* s390 has strong ordering, just use load */ + err = emit_ldx(jit, fp, insn); + if (err < 0) + return err; + break; + case BPF_STORE_REL: + /* s390 has strong ordering, just use store */ + err = emit_stx(jit, fp, insn); + if (err < 0) + return err; + break; default: pr_err("Unknown atomic operation %02x\n", insn->imm); return -1; @@ -1687,15 +1741,20 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEM | BPF_B: case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgc %dst,off(%src,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); + case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: + case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_W: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: + case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ + case BPF_LDX | BPF_PROBE_MEM | BPF_DW: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + err = emit_ldx(jit, fp, insn); if (err < 0) return err; jit->seen |= SEEN_MEM; - if (insn_is_zext(&insn[1])) + if (BPF_SIZE(insn->code) != BPF_DW && insn_is_zext(&insn[1])) insn_count = 2; break; case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */ @@ -1708,20 +1767,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, return err; jit->seen |= SEEN_MEM; break; - case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_H: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgh %dst,off(%src,%arena) */ - EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - jit->seen |= SEEN_MEM; - if (insn_is_zext(&insn[1])) - insn_count = 2; - break; case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_H: bpf_jit_probe_load_pre(jit, insn, &probe); @@ -1732,20 +1777,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, return err; jit->seen |= SEEN_MEM; break; - case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_W: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* llgf %dst,off(%src) */ - jit->seen |= SEEN_MEM; - EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - if (insn_is_zext(&insn[1])) - insn_count = 2; - break; case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */ case BPF_LDX | BPF_PROBE_MEMSX | BPF_W: bpf_jit_probe_load_pre(jit, insn, &probe); @@ -1756,18 +1787,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if (err < 0) return err; break; - case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ - case BPF_LDX | BPF_PROBE_MEM | BPF_DW: - case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: - bpf_jit_probe_load_pre(jit, insn, &probe); - /* lg %dst,off(%src,%arena) */ - jit->seen |= SEEN_MEM; - EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, - probe.arena_reg, off); - err = bpf_jit_probe_post(jit, fp, &probe); - if (err < 0) - return err; - break; /* * BPF_JMP / CALL */ @@ -3028,13 +3047,6 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) if (!in_arena) return true; switch (insn->code) { - case BPF_STX | BPF_ATOMIC | BPF_B: - case BPF_STX | BPF_ATOMIC | BPF_H: - case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (bpf_atomic_is_load_store(insn)) - return false; - break; case BPF_LDX | BPF_MEMSX | BPF_B: case BPF_LDX | BPF_MEMSX | BPF_H: case BPF_LDX | BPF_MEMSX | BPF_W: diff --git a/tools/testing/selftests/bpf/progs/arena_atomics.c b/tools/testing/selftests/bpf/progs/arena_atomics.c index 2e7751a85399..73bc2b835f3f 100644 --- a/tools/testing/selftests/bpf/progs/arena_atomics.c +++ b/tools/testing/selftests/bpf/progs/arena_atomics.c @@ -28,8 +28,10 @@ bool skip_all_tests = true; #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false; #else bool skip_lacq_srel_tests = true; @@ -315,8 +317,10 @@ int load_acquire(const void *ctx) { #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) #define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \ { asm volatile ( \ @@ -367,8 +371,10 @@ int store_release(const void *ctx) { #if defined(ENABLE_ATOMICS_TESTS) && \ defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390)) #define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \ { asm volatile ( \ diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index b0c441384f20..5eacf1b43252 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -264,9 +264,12 @@ #endif #if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \ - (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \ - (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ - defined(__TARGET_ARCH_powerpc) || defined(__TARGET_ARCH_loongarch)) + (defined(__TARGET_ARCH_arm64) || \ + defined(__TARGET_ARCH_x86) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ + defined(__TARGET_ARCH_s390) || \ + defined(__TARGET_ARCH_powerpc) || \ + defined(__TARGET_ARCH_loongarch)) #define CAN_USE_LOAD_ACQ_STORE_REL #endif