From 6bcecae4b65d80f4474c42a026a5ba229ded347e Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:30 +0000 Subject: [PATCH 1/7] bpf: Extract the is_struct_ops_tramp helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the is_struct_ops_tramp helper, and use it in riscv as the current checks are somewhat hacky. Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-2-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/arm64/net/bpf_jit_comp.c | 6 ------ arch/riscv/net/bpf_jit_comp64.c | 2 +- include/linux/bpf.h | 6 ++++++ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 0f61662b900b..4cdc7dfb05ba 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2557,12 +2557,6 @@ static void restore_args(struct jit_ctx *ctx, int bargs_off, int nregs) } } -static bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) -{ - return fentry_nodes->nr_nodes == 1 && - fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; -} - static void store_func_meta(struct jit_ctx *ctx, u64 func_meta, int func_meta_off) { emit_a64_mov_i64(A64_R(10), func_meta, ctx); diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index f9d5347ba966..406e774b4ac3 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1033,7 +1033,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; struct bpf_tramp_nodes *fmod_ret = &tnodes[BPF_TRAMP_MODIFY_RETURN]; - bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; + bool is_struct_ops = is_struct_ops_tramp(fentry); void *orig_call = func_addr; bool save_ret; u64 func_meta; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index d9542127dfdf..e066f44a9c05 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2196,6 +2196,12 @@ static inline bool is_tracing_multi(enum bpf_attach_type type) type == BPF_TRACE_FSESSION_MULTI; } +static inline bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) +{ + return fentry_nodes->nr_nodes == 1 && + fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; +} + #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) /* This macro helps developer to register a struct_ops type and generate * type information correctly. Developers should use this macro to register From 369e4635d04801f394d5bd42556f21029e95ff93 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:31 +0000 Subject: [PATCH 2/7] riscv, bpf: Fix memory leak in bpf_jit_free MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When bpf_int_jit_compile() is called for subprograms, it returns early during the first pass (!prog->is_func || extra_pass is false), keeping ctx->offset alive for the subsequent extra pass. If JIT compilation fails for a later subprogram, the BPF core aborts and calls bpf_jit_free() to clean up the first subprogram. However, bpf_jit_free() fails to free jit_data->ctx.offset, which causes a memory leak of the JIT context offsets array. Fix this by adding the missing kfree(jit_data->ctx.offset) in bpf_jit_free(). Fixes: 48a8f78c50bd ("bpf, riscv: use prog pack allocator in the BPF JIT") Reported-by: Sashiko Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-3-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index ce3bd3762e08..7cce19118619 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -234,6 +234,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); + kfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); From f3ab878594b57602aab2231d859f25ccc6b1a534 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:32 +0000 Subject: [PATCH 3/7] riscv, bpf: Using kvzalloc_objs to allocate cache buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is unnecessary to allocate continuous physical memory for cache buffer, and when ebpf program is too large, it may cause memory allocation failure. Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-4-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 4 ++-- arch/riscv/net/bpf_jit_core.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 406e774b4ac3..2f06752fb036 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -1195,7 +1195,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (fmod_ret->nr_nodes) { - branches_off = kzalloc_objs(int, fmod_ret->nr_nodes); + branches_off = kvzalloc_objs(int, fmod_ret->nr_nodes); if (!branches_off) return -ENOMEM; @@ -1300,7 +1300,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, ret = ctx->ninsns; out: - kfree(branches_off); + kvfree(branches_off); return ret; } diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 7cce19118619..cbfcd287ea16 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -72,7 +72,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); ctx->prog = prog; - ctx->offset = kzalloc_objs(int, prog->len); + ctx->offset = kvzalloc_objs(int, prog->len); if (!ctx->offset) goto out_offset; @@ -170,7 +170,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); out_offset: - kfree(ctx->offset); + kvfree(ctx->offset); kfree(jit_data); prog->aux->jit_data = NULL; } @@ -234,7 +234,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); - kfree(jit_data->ctx.offset); + kvfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); From 52fb1756ea1d2759dfef2d86245be00b05dac3a2 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:33 +0000 Subject: [PATCH 4/7] riscv, bpf: Fix kernel stack corruption in tailcall with CFI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When CONFIG_CFI_CLANG is enabled, prog->bpf_func already skips the kcfi instruction during setup. Including it again in the tailcall jump offset causes it to jump over an extra 4 bytes, skipping the stack pointer adjustment, which will result in kernel stack corruption. Fixes: 30a59cc79754 ("riscv, bpf: Fix possible infinite tailcall when CONFIG_CFI_CLANG is enabled") Reported-by: Sashiko Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-5-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 2f06752fb036..cedf40927474 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -18,7 +18,6 @@ #define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) -#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF @@ -272,8 +271,8 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) if (!is_tail_call) emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - /* kcfi, fentry and TCC init insns will be skipped on tailcall */ - is_tail_call ? (RV_KCFI_NINSNS + RV_FENTRY_NINSNS + 1) * 4 : 0, + /* fentry and TCC init insns will be skipped on tailcall */ + is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, ctx); } @@ -2033,6 +2032,8 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) /* emit kcfi type preamble immediately before the first insn */ emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* bpf prog starts here as kcfi skipped during prog->bpf_func setup */ + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); From a21731f54cfe59733357d8e0dc932f6bf2fcee2f Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:34 +0000 Subject: [PATCH 5/7] riscv, bpf: Add RV_TAILCALL_OFFSET macro to format tailcall offset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add RV_TAILCALL_OFFSET macro to format tailcall offset, and correct the relevant comments. Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-6-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index cedf40927474..7c6304e0b846 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -20,6 +20,8 @@ #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF +/* fentry and TCC init insns will be skipped on tailcall */ +#define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) #define RV_REG_TCC RV_REG_A6 #define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ @@ -271,9 +273,7 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) if (!is_tail_call) emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - /* fentry and TCC init insns will be skipped on tailcall */ - is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, - ctx); + is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx); } static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, @@ -393,7 +393,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); - /* goto *(prog->bpf_func + 4); */ + /* goto *(prog->bpf_func + RV_TAILCALL_OFFSET); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) return -1; From ec72848ca0add4aff1e690873c7cd8b5eef4bcfd Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:35 +0000 Subject: [PATCH 6/7] riscv, bpf: Mixing bpf2bpf and tailcalls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the current RV64 JIT, if we just don't initialize the TCC in subprog, the TCC can be propagated from the parent process to the subprocess, but the updated TCC of the parent process cannot be restored when the subprocess exits. Since the RV64 TCC is initialized before saving the callee saved registers into the stack, we cannot use the callee saved register to pass the TCC, otherwise the original value of the callee saved register will be destroyed. So we implemented mixing bpf2bpf and tailcalls similar to x86_64, i.e. using a non-callee saved register to transfer the TCC between functions, and saving that register to the stack to protect the TCC value. As for the tailcall hierarchy issue, inspired by the s390's low-overhead approach, we store TCC from RV_REG_TCC back to stack after calling bpf2bpf call or calling orig bpf func in bpf trampoline. Tests test_bpf.ko and test_verifier have passed, as well as the relative testcases of test_progs*. Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-7-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit.h | 1 + arch/riscv/net/bpf_jit_comp64.c | 108 ++++++++++++++++---------------- 2 files changed, 54 insertions(+), 55 deletions(-) diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index da0271790244..419b9d795f2a 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -81,6 +81,7 @@ struct rv_jit_context { int ex_jmp_off; unsigned long flags; int stack_size; + int tcc_offset; u64 arena_vm_start; u64 user_vm_start; }; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 7c6304e0b846..823262ca47eb 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -24,7 +24,6 @@ #define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) #define RV_REG_TCC RV_REG_A6 -#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */ #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ static const int regmap[] = { @@ -58,14 +57,12 @@ static const int pt_regmap[] = { }; enum { - RV_CTX_F_SEEN_TAIL_CALL = 0, RV_CTX_F_SEEN_CALL = RV_REG_RA, RV_CTX_F_SEEN_S1 = RV_REG_S1, RV_CTX_F_SEEN_S2 = RV_REG_S2, RV_CTX_F_SEEN_S3 = RV_REG_S3, RV_CTX_F_SEEN_S4 = RV_REG_S4, RV_CTX_F_SEEN_S5 = RV_REG_S5, - RV_CTX_F_SEEN_S6 = RV_REG_S6, }; static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) @@ -78,7 +75,6 @@ static u8 bpf_to_rv_reg(int bpf_reg, struct rv_jit_context *ctx) case RV_CTX_F_SEEN_S3: case RV_CTX_F_SEEN_S4: case RV_CTX_F_SEEN_S5: - case RV_CTX_F_SEEN_S6: __set_bit(reg, &ctx->flags); } return reg; @@ -93,7 +89,6 @@ static bool seen_reg(int reg, struct rv_jit_context *ctx) case RV_CTX_F_SEEN_S3: case RV_CTX_F_SEEN_S4: case RV_CTX_F_SEEN_S5: - case RV_CTX_F_SEEN_S6: return test_bit(reg, &ctx->flags); } return false; @@ -109,32 +104,6 @@ static void mark_call(struct rv_jit_context *ctx) __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); } -static bool seen_call(struct rv_jit_context *ctx) -{ - return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); -} - -static void mark_tail_call(struct rv_jit_context *ctx) -{ - __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); -} - -static bool seen_tail_call(struct rv_jit_context *ctx) -{ - return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); -} - -static u8 rv_tail_call_reg(struct rv_jit_context *ctx) -{ - mark_tail_call(ctx); - - if (seen_call(ctx)) { - __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); - return RV_REG_S6; - } - return RV_REG_A6; -} - static bool is_32b_int(s64 val) { return -(1L << 31) <= val && val < (1L << 31); @@ -259,15 +228,14 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) emit_ld(RV_REG_S5, store_offset, RV_REG_SP, ctx); store_offset -= 8; } - if (seen_reg(RV_REG_S6, ctx)) { - emit_ld(RV_REG_S6, store_offset, RV_REG_SP, ctx); - store_offset -= 8; - } if (ctx->arena_vm_start) { emit_ld(RV_REG_ARENA, store_offset, RV_REG_SP, ctx); store_offset -= 8; } + /* restore TCC from stack to RV_REG_TCC */ + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) @@ -354,7 +322,6 @@ static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) { int tc_ninsn, off, start_insn = ctx->ninsns; - u8 tcc = rv_tail_call_reg(ctx); /* a0: &ctx * a1: &array @@ -377,7 +344,8 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) /* if (--TCC < 0) * goto out; */ - emit_addi(RV_REG_TCC, tcc, -1, ctx); + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + emit_addi(RV_REG_TCC, RV_REG_TCC, -1, ctx); off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); @@ -393,6 +361,9 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); + /* store updated TCC back to stack */ + emit_sd(RV_REG_SP, ctx->tcc_offset, RV_REG_TCC, ctx); + /* goto *(prog->bpf_func + RV_TAILCALL_OFFSET); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) @@ -1027,7 +998,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int i, ret, offset; int *branches_off = NULL; int stack_size = 0, nr_arg_slots = 0; - int retval_off, args_off, func_meta_off, ip_off, run_ctx_off, sreg_off, stk_arg_off; + int retval_off, args_off, func_meta_off, ip_off; + int run_ctx_off, sreg_off, stk_arg_off, tcc_off; int cookie_off, cookie_cnt; struct bpf_tramp_nodes *fentry = &tnodes[BPF_TRAMP_FENTRY]; struct bpf_tramp_nodes *fexit = &tnodes[BPF_TRAMP_FEXIT]; @@ -1078,6 +1050,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, * * FP - sreg_off [ callee saved reg ] * + * FP - tcc_off [ tail call count ] BPF_TRAMP_F_TAIL_CALL_CTX + * * [ pads ] pads for 16 bytes alignment * * [ stack_argN ] @@ -1125,6 +1099,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, stack_size += 8; sreg_off = stack_size; + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_off = stack_size; + } + if ((flags & BPF_TRAMP_F_CALL_ORIG) && (nr_arg_slots - RV_MAX_REG_ARGS > 0)) stack_size += (nr_arg_slots - RV_MAX_REG_ARGS) * 8; @@ -1159,6 +1138,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); } + /* store tail call count */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); + /* callee saved register S1 to pass start time */ emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); @@ -1217,9 +1200,15 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, orig_call += RV_FENTRY_NINSNS * 4; restore_args(min_t(int, nr_arg_slots, RV_MAX_REG_ARGS), args_off, ctx); restore_stack_args(nr_arg_slots - RV_MAX_REG_ARGS, args_off, stk_arg_off, ctx); + /* restore TCC to RV_REG_TCC before calling the orig bpf func */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_ld(RV_REG_TCC, -tcc_off, RV_REG_FP, ctx); ret = emit_call((const u64)orig_call, true, ctx); if (ret) goto out; + /* store updated TCC back to stack after calling the orig bpf func */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); im->ip_after_call = ctx->ro_insns + ctx->ninsns; @@ -1272,6 +1261,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); + /* restore TCC from stack to RV_REG_TCC */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_ld(RV_REG_TCC, -tcc_off, RV_REG_FP, ctx); + if (!is_struct_ops) { /* trampoline called from function entry */ emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); @@ -1836,10 +1829,18 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, } } + /* restore TCC to RV_REG_TCC before bpf2bpf call */ + if (aux->tail_call_reachable && insn->src_reg == BPF_PSEUDO_CALL) + emit_ld(RV_REG_TCC, ctx->tcc_offset, RV_REG_SP, ctx); + ret = emit_call(addr, fixed_addr, ctx); if (ret) return ret; + /* store updated TCC back to stack after bpf2bpf call */ + if (aux->tail_call_reachable && insn->src_reg == BPF_PSEUDO_CALL) + emit_sd(RV_REG_SP, ctx->tcc_offset, RV_REG_TCC, ctx); + if (insn->src_reg != BPF_PSEUDO_CALL) emit_mv(bpf_to_rv_reg(BPF_REG_0, ctx), RV_REG_A0, ctx); break; @@ -2019,10 +2020,9 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) stack_adjust += 8; if (seen_reg(RV_REG_S5, ctx)) stack_adjust += 8; - if (seen_reg(RV_REG_S6, ctx)) - stack_adjust += 8; if (ctx->arena_vm_start) stack_adjust += 8; + stack_adjust += 8; /* RV_REG_TCC */ stack_adjust = round_up(stack_adjust, STACK_ALIGN); stack_adjust += bpf_stack_adjust; @@ -2038,11 +2038,10 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); - /* First instruction is always setting the tail-call-counter - * (TCC) register. This instruction is skipped for tail calls. - * Force using a 4-byte (non-compressed) instruction. - */ - emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); + if (!is_subprog) + emit(rv_addi(RV_REG_TCC, RV_REG_ZERO, MAX_TAIL_CALL_CNT), ctx); + + /* tailcall starts here, emit insn before it must be fixed */ emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); @@ -2072,26 +2071,20 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) emit_sd(RV_REG_SP, store_offset, RV_REG_S5, ctx); store_offset -= 8; } - if (seen_reg(RV_REG_S6, ctx)) { - emit_sd(RV_REG_SP, store_offset, RV_REG_S6, ctx); - store_offset -= 8; - } if (ctx->arena_vm_start) { emit_sd(RV_REG_SP, store_offset, RV_REG_ARENA, ctx); store_offset -= 8; } + /* store TCC from RV_REG_TCC to stack */ + emit_sd(RV_REG_SP, store_offset, RV_REG_TCC, ctx); + ctx->tcc_offset = store_offset; + emit_addi(RV_REG_FP, RV_REG_SP, stack_adjust, ctx); if (bpf_stack_adjust) emit_addi(RV_REG_S5, RV_REG_SP, bpf_stack_adjust, ctx); - /* Program contains calls and tail calls, so RV_REG_TCC need - * to be saved across calls. - */ - if (seen_tail_call(ctx) && seen_call(ctx)) - emit_mv(RV_REG_TCC_SAVED, RV_REG_TCC, ctx); - ctx->stack_size = stack_adjust; if (ctx->arena_vm_start) @@ -2158,3 +2151,8 @@ bool bpf_jit_supports_fsession(void) { return true; } + +bool bpf_jit_supports_subprog_tailcalls(void) +{ + return true; +} From 683ed8b78c1d03bdddb17188b45ea803761d6d5f Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 8 Jul 2026 06:44:36 +0000 Subject: [PATCH 7/7] selftests/bpf: Remove tailcalls tests from DENYLIST.riscv64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RV64 BPF JIT now supports mixing bpf2bpf and tailcalls. Therefore, the tailcall_bpf2bpf tests can be safely removed from the riscv64 denylist. Signed-off-by: Pu Lehui Reviewed-by: Björn Töpel Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260708064436.2971933-8-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/DENYLIST.riscv64 | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/bpf/DENYLIST.riscv64 b/tools/testing/selftests/bpf/DENYLIST.riscv64 index 4fc4dfdde293..ca1beae7fe8f 100644 --- a/tools/testing/selftests/bpf/DENYLIST.riscv64 +++ b/tools/testing/selftests/bpf/DENYLIST.riscv64 @@ -1,3 +1,2 @@ # riscv64 deny list for BPF CI and local vmtest exceptions # JIT does not support exceptions -tailcalls/tailcall_bpf2bpf* # JIT does not support mixing bpf2bpf and tailcalls