diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 9525d3ecd923..1e1ce2afe2ed 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1833,7 +1833,6 @@ struct bpf_prog_aux { bool xdp_has_frags; bool exception_cb; bool exception_boundary; - bool is_extended; /* true if extended by freplace program */ bool jits_use_priv_stack; bool priv_stack_requested; bool changes_pkt_data; @@ -1845,7 +1844,7 @@ struct bpf_prog_aux { u8 verdict; } sig; u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ - struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ + struct mutex ext_mutex; /* mutex for freplace_link_cnt and prog_array_member_cnt */ struct bpf_arena *arena; void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */ /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ @@ -1877,6 +1876,7 @@ struct bpf_prog_aux { char name[BPF_OBJ_NAME_LEN]; u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); u16 stack_arg_sp_adjust; + u16 freplace_link_cnt; /* counts freplace links extending this prog */ #ifdef CONFIG_SECURITY void *security; #endif diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 0ce26b538075..0fe9afd4a591 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -1035,7 +1035,7 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, } mutex_lock(&prog->aux->ext_mutex); - is_extended = prog->aux->is_extended; + is_extended = prog->aux->freplace_link_cnt > 0; if (!is_extended) prog->aux->prog_array_member_cnt++; mutex_unlock(&prog->aux->ext_mutex); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 90b70ea0d370..da85bd580ef0 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -843,7 +843,7 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) } } -static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) +static int bpf_freplace_link_tgt_prog(struct bpf_prog *tgt_prog) { struct bpf_prog_aux *aux = tgt_prog->aux; @@ -857,7 +857,7 @@ static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog) */ return -EBUSY; - aux->is_extended = true; + aux->freplace_link_cnt++; return 0; } @@ -970,7 +970,7 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, /* Cannot attach extension if fentry/fexit are in use. */ if (cnt) return -EBUSY; - err = bpf_freplace_check_tgt_prog(tgt_prog); + err = bpf_freplace_link_tgt_prog(tgt_prog); if (err) return err; tr->extension_prog = node->link->prog; @@ -1016,7 +1016,7 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, tr->extension_prog->bpf_func, NULL); tr->extension_prog = NULL; guard(mutex)(&tgt_prog->aux->ext_mutex); - tgt_prog->aux->is_extended = false; + tgt_prog->aux->freplace_link_cnt--; return err; } bpf_trampoline_remove_prog(tr, node); diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index c5c9d6c359bb..aefb46778307 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -6,6 +6,7 @@ #include "tailcall_bpf2bpf_hierarchy2.skel.h" #include "tailcall_bpf2bpf_hierarchy3.skel.h" #include "tailcall_freplace.skel.h" +#include "tailcall_freplace_multi.skel.h" #include "tc_bpf2bpf.skel.h" #include "tailcall_fail.skel.h" #include "tailcall_cgrp_storage_owner.skel.h" @@ -1655,6 +1656,77 @@ static void test_tailcall_bpf2bpf_freplace(void) tc_bpf2bpf__destroy(tc_skel); } +static void test_tailcall_freplace_multi(void) +{ + struct tailcall_freplace_multi *freplace_skel = NULL; + struct bpf_link *link_subprog = NULL, *link_entry = NULL; + struct tc_bpf2bpf *tc_skel = NULL; + int tc_prog_fd, map_fd, key = 0, err; + + tc_skel = tc_bpf2bpf__open_and_load(); + if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load")) + return; + + tc_prog_fd = bpf_program__fd(tc_skel->progs.entry_tc); + + freplace_skel = tailcall_freplace_multi__open(); + if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace_multi__open")) + goto out; + + err = bpf_program__set_attach_target(freplace_skel->progs.subprog_freplace, + tc_prog_fd, "subprog_tc"); + if (!ASSERT_OK(err, "set_attach_target subprog_tc")) + goto out; + + err = bpf_program__set_attach_target(freplace_skel->progs.entry_freplace, + tc_prog_fd, "entry_tc"); + if (!ASSERT_OK(err, "set_attach_target entry_tc")) + goto out; + + err = tailcall_freplace_multi__load(freplace_skel); + if (!ASSERT_OK(err, "tailcall_freplace_multi__load")) + goto out; + + map_fd = bpf_map__fd(freplace_skel->maps.jmp_table); + + link_subprog = bpf_program__attach_freplace(freplace_skel->progs.subprog_freplace, + tc_prog_fd, "subprog_tc"); + if (!ASSERT_OK_PTR(link_subprog, "attach_freplace subprog_tc")) + goto out; + + link_entry = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace, + tc_prog_fd, "entry_tc"); + if (!ASSERT_OK_PTR(link_entry, "attach_freplace entry_tc")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + if (!ASSERT_ERR(err, "update jmp_table with extended prog")) + goto out; + + err = bpf_link__destroy(link_entry); + link_entry = NULL; + if (!ASSERT_OK(err, "destroy entry link")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + if (!ASSERT_ERR(err, "update jmp_table with still extended prog")) + goto out; + + err = bpf_link__destroy(link_subprog); + link_subprog = NULL; + if (!ASSERT_OK(err, "destroy subprog link")) + goto out; + + err = bpf_map_update_elem(map_fd, &key, &tc_prog_fd, BPF_ANY); + ASSERT_OK(err, "update jmp_table"); + +out: + bpf_link__destroy(link_subprog); + bpf_link__destroy(link_entry); + tailcall_freplace_multi__destroy(freplace_skel); + tc_bpf2bpf__destroy(tc_skel); +} + static void test_tailcall_failure() { RUN_TESTS(tailcall_fail); @@ -2005,6 +2077,8 @@ void test_tailcalls(void) test_tailcall_freplace(); if (test__start_subtest("tailcall_bpf2bpf_freplace")) test_tailcall_bpf2bpf_freplace(); + if (test__start_subtest("tailcall_freplace_multi")) + test_tailcall_freplace_multi(); if (test__start_subtest("tailcall_failure")) test_tailcall_failure(); if (test__start_subtest("tailcall_sleepable")) diff --git a/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c b/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c new file mode 100644 index 000000000000..ca764411dc0b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tailcall_freplace_multi.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 KylinSoft Corporation. */ + +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u32)); +} jmp_table SEC(".maps"); + +SEC("freplace") +int subprog_freplace(struct __sk_buff *skb) +{ + bpf_tail_call_static(skb, &jmp_table, 0); + return 0; +} + +SEC("freplace") +int entry_freplace(struct __sk_buff *skb) +{ + return 0; +} + +char __license[] SEC("license") = "GPL";