github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

riscv, bpf: Add BPF stack arguments support for RV64 JIT

Browse Source 
Implement bpf_jit_supports_stack_args() on RV64 JIT to allow bpf
subprogs and kfuncs to pass and receive more than 5 arguments.

In the riscv abi, the first 8 arguments are passed in registers a0 to
a7, and arguments 9+ reside on the stack. To align bpf stack arguments
with this calling convention and unify bpf2bpf calls with kfuncs, map
the first 3 bpf stack arguments (6th to 8th) directly to a5 to a7, and
store or load the remaining arguments (9th+) at SP or FP. Reserve
outgoing stack space in the prologue accordingly when stack_arg_cnt
exceeds 3.

In addition, update kfunc argument sign extension to handle all 8
register arguments as well as any arguments passed on the stack.

Co-developed-by: Pu Lehui <pulehui@huawei.com>
Signed-off-by: Feng Jiang <jiangfeng@kylinos.cn>
Signed-off-by: Pu Lehui <pulehui@huawei.com>
Acked-by: Björn Töpel <bjorn@kernel.org>
Link: https://lore.kernel.org/bpf/20260905085857.14143-3-pulehui@huaweicloud.com
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
This commit is contained in:
Feng Jiang 2026-09-05 08:58:56 +00:00 committed by Kumar Kartikeya Dwivedi
parent f80a09c486
commit ba754185f9
No known key found for this signature in database
GPG Key ID: 472D377B63542F83
3 changed files with 80 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
arch/riscv/net/bpf_jit.h
View File

@ -12,6 +12,9 @@
#include <linux/bpf.h>
#include <linux/filter.h>
#define RV_MAX_REG_ARGS 8
#define RV_EXTRA_STK_ARGS (RV_MAX_REG_ARGS - MAX_BPF_FUNC_REG_ARGS)
/* verify runtime detection extension status */
#define rv_ext_enabled(ext) \
(IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext))
@ -82,6 +85,7 @@ struct rv_jit_context {
unsigned long flags;
int stack_size;
int tcc_offset;
int stack_arg_sz;
u64 arena_vm_start;
u64 user_vm_start;
};

78
arch/riscv/net/bpf_jit_comp64.c
View File

@ -15,7 +15,6 @@
#include <asm/percpu.h>
#include "bpf_jit.h"
#define RV_MAX_REG_ARGS 8
#define RV_FENTRY_NINSNS 2
#define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4)
/* imm that allows emit_imm to emit max count insns */
@ -498,6 +497,18 @@ static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext,
ctx->ex_jmp_off = ctx->ninsns;
}
static void emit_stack_arg_ldx(u8 rd, s16 off, struct rv_jit_context *ctx)
{
int idx = off / 8 - 1;
if (idx < RV_EXTRA_STK_ARGS) {
emit_mv(rd, RV_REG_A5 + idx, ctx);
return;
}
emit_ldx_insn(rd, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_FP, BPF_DW, false, ctx);
}
static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx)
{
emit_imm(RV_REG_T1, imm, ctx);
@ -515,6 +526,19 @@ static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx
ctx->ex_jmp_off = ctx->ninsns;
}
static void emit_stack_arg_st(s16 off, s32 imm, struct rv_jit_context *ctx)
{
int idx = -off / 8 - 1;
if (idx < RV_EXTRA_STK_ARGS) {
emit_imm(RV_REG_A5 + idx, imm, ctx);
return;
}
emit_imm(RV_REG_T1, imm, ctx);
emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_T1, BPF_DW, ctx);
}
static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx)
{
if (is_12b_int(off)) {
@ -531,6 +555,18 @@ static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx)
ctx->ex_jmp_off = ctx->ninsns;
}
static void emit_stack_arg_stx(s16 off, u8 rs, struct rv_jit_context *ctx)
{
int idx = -off / 8 - 1;
if (idx < RV_EXTRA_STK_ARGS) {
emit_mv(RV_REG_A5 + idx, rs, ctx);
return;
}
emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, rs, BPF_DW, ctx);
}
static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn,
struct rv_jit_context *ctx)
{
@ -1824,11 +1860,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
return -EINVAL;
for (idx = 0; idx < fm->nr_args; idx++) {
u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx);
bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG;
u8 arg_sz = fm->arg_size[idx];
if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx))
return -EINVAL;
if (arg_sz == 8 || (arg_sz != 4 && !sign))
continue;
if (idx < RV_MAX_REG_ARGS) {
if (sign_extend(RV_REG_A0 + idx, RV_REG_A0 + idx, arg_sz, sign, ctx))
return -EINVAL;
} else {
emit_ld(RV_REG_T1, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_SP, ctx);
if (sign_extend(RV_REG_T1, RV_REG_T1, arg_sz, sign, ctx))
return -EINVAL;
emit_sd(RV_REG_SP, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_T1, ctx);
}
}
}
@ -1927,7 +1973,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
rs = RV_REG_T2;
}
emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx);
if (is_stack_arg_ldx(insn))
emit_stack_arg_ldx(rd, off, ctx);
else
emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx);
ret = add_exception_handler(insn, rd, ctx);
if (ret)
@ -1957,7 +2006,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
rd = RV_REG_T3;
}
emit_st(rd, off, imm, BPF_SIZE(code), ctx);
if (is_stack_arg_st(insn))
emit_stack_arg_st(off, imm, ctx);
else
emit_st(rd, off, imm, BPF_SIZE(code), ctx);
ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
if (ret)
@ -1979,7 +2031,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
rd = RV_REG_T2;
}
emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
if (is_stack_arg_stx(insn))
emit_stack_arg_stx(off, rs, ctx);
else
emit_stx(rd, off, rs, BPF_SIZE(code), ctx);
ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx);
if (ret)
@ -2029,9 +2084,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog)
{
int i, stack_adjust = 0, store_offset, bpf_stack_adjust;
int i, stack_adjust = 0, store_offset, bpf_stack_adjust = ctx->stack_arg_sz;
bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN);
bpf_stack_adjust += round_up(ctx->prog->aux->stack_depth, STACK_ALIGN);
if (bpf_stack_adjust)
mark_fp(ctx);
@ -2190,3 +2245,8 @@ bool bpf_jit_supports_timed_may_goto(void)
{
return true;
}
bool bpf_jit_supports_stack_args(void)
{
return true;
}

7
arch/riscv/net/bpf_jit_core.c
View File

@ -48,6 +48,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
int pass = 0, prev_ninsns = 0, i;
struct rv_jit_data *jit_data;
struct rv_jit_context *ctx;
u16 stack_arg_cnt;
if (!prog->jit_requested)
return prog;
@ -71,6 +72,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog);
/* First 3 stack args in regs, rest on stack */
ctx->stack_arg_sz = stack_arg_cnt > RV_EXTRA_STK_ARGS ?
round_up((stack_arg_cnt - RV_EXTRA_STK_ARGS) * 8, STACK_ALIGN) : 0;
ctx->prog = prog;
ctx->offset = kvzalloc_objs(int, prog->len);
if (!ctx->offset)