diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 0f61662b900b..4cdc7dfb05ba 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2557,12 +2557,6 @@ static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs) } } -static bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) -{ - return fentry_nodes->nr_nodes == 1 && - fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; -} - static void store_func_meta(struct jit_ctx *ctx, u64 func_meta, int func_meta_off) { emit_a64_mov_i64(A64_R(10), func_meta, ctx); diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index da0271790244..419b9d795f2a 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -81,6 +81,7 @@ struct rv_jit_context { int ex_jmp_off; unsigned long flags; int stack_size; + int tcc_offset; u64 arena_vm_start; u64 user_vm_start; }; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index f9d5347ba966..823262ca47eb 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -18,12 +18,12 @@ #define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) -#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF +/* fentry and TCC init insns will be skipped on tailcall */ +#define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) #define RV_REG_TCC RV_REG_A6 -#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ static const int regmap[] = { @@ -57,14 +57,12 @@ static const int pt_regmap[] = { }; enum { - RV_CTX_F_SEEN_TAIL_CALL = 0, RV_CTX_F_SEEN_CALL = RV_REG_RA, RV_CTX_F_SEEN_S1 = RV_REG_S1, RV_CTX_F_SEEN_S2 = RV_REG_S2, RV_CTX_F_SEEN_S3 = RV_REG_S3, RV_CTX_F_SEEN_S4 = RV_REG_S4, RV_CTX_F_SEEN_S5 = RV_REG_S5, - RV_CTX_F_SEEN_S6 = RV_REG_S6, }; static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) @@ -77,7 +75,6 @@ static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) case RV_CTX_F_SEEN_S3: case RV_CTX_F_SEEN_S4: case RV_CTX_F_SEEN_S5: - case RV_CTX_F_SEEN_S6: __set_bit(reg, &ctx->flags); } return reg; @@ -92,7 +89,6 @@ static bool seen_reg(int reg, struct rv_jit_context *ctx) case RV_CTX_F_SEEN_S3: case RV_CTX_F_SEEN_S4: case RV_CTX_F_SEEN_S5: - case RV_CTX_F_SEEN_S6: return test_bit(reg, &ctx->flags); } return false; @@ -108,32 +104,6 @@ static void mark_call(struct rv_jit_context *ctx) __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); } -static bool seen_call(struct rv_jit_context *ctx) -{ - return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); -} - -static void mark_tail_call(struct rv_jit_context *ctx) -{ - __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); -} - -static bool seen_tail_call(struct rv_jit_context *ctx) -{ - return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); -} - -static u8 rv_tail_call_reg(struct rv_jit_context *ctx) -{ - mark_tail_call(ctx); - - if (seen_call(ctx)) { - __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); - return RV_REG_S6; - } - return RV_REG_A6; -} - static bool is_32b_int(s64 val) { return -(1L << 31) <= val && val < (1L << 31); @@ -258,23 +228,20 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); store_offset -= 8; } - if (seen_reg(RV_REG_S6, ctx)) { - emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); - store_offset -= 8; - } if (ctx->arena_vm_start) { emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx); store_offset -= 8; } + /* restore TCC from stack to RV_REG_TCC */ + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - /* kcfi, fentry and TCC init insns will be skipped on tailcall */ - is_tail_call ? (RV_KCFI_NINSNS + RV_FENTRY_NINSNS + 1) * 4 : 0, - ctx); + is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx); } static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, @@ -355,7 +322,6 @@ static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) { int tc_ninsn, off, start_insn = ctx->ninsns; - u8 tcc = rv_tail_call_reg(ctx); /* a0: &ctx * a1: &array @@ -378,7 +344,8 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) /* if (--TCC < 0) * goto out; */ - emit_addi(RV_REG_TCC, tcc, -1, ctx); + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + emit_addi(RV_REG_TCC, RV_REG_TCC, -1, ctx); off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); @@ -394,7 +361,10 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); - /* goto *(prog->bpf_func + 4); */ + /* store updated TCC back to stack */ + emit_sd(RV_REG_SP, ctx->tcc_offset, RV_REG_TCC, ctx); + + /* goto *(prog->bpf_func + RV_TAILCALL_OFFSET); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) return -1; @@ -1028,12 +998,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int i, ret, offset; int *branches_off = NULL; int stack_size = 0, nr_arg_slots = 0; - int retval_off, args_off, func_meta_off, ip_off, run_ctx_off, sreg_off, stk_arg_off; + int retval_off, args_off, func_meta_off, ip_off; + int run_ctx_off, sreg_off, stk_arg_off, tcc_off; int cookie_off, cookie_cnt; struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; - bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; + bool is_struct_ops = is_struct_ops_tramp(fentry); void *orig_call = func_addr; bool save_ret; u64 func_meta; @@ -1079,6 +1050,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, * * FP - sreg_off [ callee saved reg ] * + * FP - tcc_off [ tail call count ] BPF_TRAMP_F_TAIL_CALL_CTX + * * [ pads ] pads for 16 bytes alignment * * [ stack_argN ] @@ -1126,6 +1099,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, stack_size += 8; sreg_off = stack_size; + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_off = stack_size; + } + if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0)) stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8; @@ -1160,6 +1138,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); } + /* store tail call count */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); + /* callee saved register S1 to pass start time */ emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); @@ -1195,7 +1177,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (fmod_ret->nr_nodes) { - branches_off = kzalloc_objs(int, fmod_ret->nr_nodes); + branches_off = kvzalloc_objs(int, fmod_ret->nr_nodes); if (!branches_off) return -ENOMEM; @@ -1218,9 +1200,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, orig_call += RV_FENTRY_NINSNS * 4; restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx); + /* restore TCC to RV_REG_TCC before calling the orig bpf func */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_ld(RV_REG_TCC, -tcc_off, RV_REG_FP, ctx); ret = emit_call((const u64)orig_call, true, ctx); if (ret) goto out; + /* store updated TCC back to stack after calling the orig bpf func */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); im->ip_after_call = ctx->ro_insns + ctx->ninsns; @@ -1273,6 +1261,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); + /* restore TCC from stack to RV_REG_TCC */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_ld(RV_REG_TCC, -tcc_off, RV_REG_FP, ctx); + if (!is_struct_ops) { /* trampoline called from function entry */ emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); @@ -1300,7 +1292,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, ret = ctx->ninsns; out: - kfree(branches_off); + kvfree(branches_off); return ret; } @@ -1837,10 +1829,18 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, } } + /* restore TCC to RV_REG_TCC before bpf2bpf call */ + if (aux->tail_call_reachable && insn->src_reg == BPF_PSEUDO_CALL) + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + ret = emit_call(addr, fixed_addr, ctx); if (ret) return ret; + /* store updated TCC back to stack after bpf2bpf call */ + if (aux->tail_call_reachable && insn->src_reg == BPF_PSEUDO_CALL) + emit_sd(RV_REG_SP, ctx->tcc_offset, RV_REG_TCC, ctx); + if (insn->src_reg != BPF_PSEUDO_CALL) emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); break; @@ -2020,10 +2020,9 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) stack_adjust += 8; if (seen_reg(RV_REG_S5, ctx)) stack_adjust += 8; - if (seen_reg(RV_REG_S6, ctx)) - stack_adjust += 8; if (ctx->arena_vm_start) stack_adjust += 8; + stack_adjust += 8; /* RV_REG_TCC */ stack_adjust = round_up(stack_adjust, STACK_ALIGN); stack_adjust += bpf_stack_adjust; @@ -2033,15 +2032,16 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) /* emit kcfi type preamble immediately before the first insn */ emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* bpf prog starts here as kcfi skipped during prog->bpf_func setup */ + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); - /* First instruction is always setting the tail-call-counter - * (TCC) register. This instruction is skipped for tail calls. - * Force using a 4-byte (non-compressed) instruction. - */ - emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); + if (!is_subprog) + emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); + + /* tailcall starts here, emit insn before it must be fixed */ emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); @@ -2071,26 +2071,20 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); store_offset -= 8; } - if (seen_reg(RV_REG_S6, ctx)) { - emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); - store_offset -= 8; - } if (ctx->arena_vm_start) { emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx); store_offset -= 8; } + /* store TCC from RV_REG_TCC to stack */ + emit_sd(RV_REG_SP, store_offset, RV_REG_TCC, ctx); + ctx->tcc_offset = store_offset; + emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); if (bpf_stack_adjust) emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); - /* Program contains calls and tail calls, so RV_REG_TCC need - * to be saved across calls. - */ - if (seen_tail_call(ctx) && seen_call(ctx)) - emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); - ctx->stack_size = stack_adjust; if (ctx->arena_vm_start) @@ -2157,3 +2151,8 @@ bool bpf_jit_supports_fsession(void) { return true; } + +bool bpf_jit_supports_subprog_tailcalls(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index ce3bd3762e08..cbfcd287ea16 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -72,7 +72,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); ctx->prog = prog; - ctx->offset = kzalloc_objs(int, prog->len); + ctx->offset = kvzalloc_objs(int, prog->len); if (!ctx->offset) goto out_offset; @@ -170,7 +170,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); out_offset: - kfree(ctx->offset); + kvfree(ctx->offset); kfree(jit_data); prog->aux->jit_data = NULL; } @@ -234,6 +234,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); + kvfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d9542127dfdf..e066f44a9c05 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2196,6 +2196,12 @@ static inline bool is_tracing_multi(enum bpf_attach_type type) type == BPF_TRACE_FSESSION_MULTI; } +static inline bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) +{ + return fentry_nodes->nr_nodes == 1 && + fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; +} + #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) /* This macro helps developer to register a struct_ops type and generate * type information correctly. Developers should use this macro to register diff --git a/tools/testing/selftests/bpf/DENYLIST.riscv64 b/tools/testing/selftests/bpf/DENYLIST.riscv64 index 4fc4dfdde293..ca1beae7fe8f 100644 --- a/tools/testing/selftests/bpf/DENYLIST.riscv64 +++ b/tools/testing/selftests/bpf/DENYLIST.riscv64 @@ -1,3 +1,2 @@ # riscv64 deny list for BPF CI and local vmtest exceptions # JIT does not support exceptions -tailcalls/tailcall_bpf2bpf* # JIT does not support mixing bpf2bpf and tailcalls