diff --git a/include/linux/btf.h b/include/linux/btf.h index c09b7994de4e..3f5255d095a2 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -79,6 +79,7 @@ #define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */ #define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ +#define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 555ee2531ef9..7b6847200b43 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -1118,9 +1118,9 @@ __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_c __bpf_kfunc_end_defs(); BTF_KFUNCS_START(arena_kfuncs) -BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2) -BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2) -BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2) +BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_ARENA_RET | KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_arena_reserve_pages, KF_ARENA_ARG2 | KF_SPINLOCK_SAFE) BTF_KFUNCS_END(arena_kfuncs) static const struct btf_kfunc_id_set common_kfunc_set = { diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 4709a5ad0474..6388b6b23e49 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -4812,30 +4812,32 @@ BTF_ID_FLAGS(func, bpf_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_obj_drop_impl, KF_RELEASE) BTF_ID_FLAGS(func, bpf_percpu_obj_drop, KF_RELEASE | KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_percpu_obj_drop_impl, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_refcount_acquire, KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, KF_ACQUIRE | KF_RET_NULL | KF_RCU) -BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_push_front_impl) -BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_push_back_impl) -BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_list_is_first) -BTF_ID_FLAGS(func, bpf_list_is_last) -BTF_ID_FLAGS(func, bpf_list_empty) +BTF_ID_FLAGS(func, bpf_refcount_acquire, + KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_refcount_acquire_impl, + KF_ACQUIRE | KF_RET_NULL | KF_RCU | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_front, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_front_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_back, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_push_back_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_add, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_pop_front, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_is_first, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_is_last, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_list_empty, KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE) -BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_rbtree_add_impl) -BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_add, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_add_impl, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_first, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_root, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_left, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_rbtree_right, KF_RET_NULL | KF_SPINLOCK_SAFE) #ifdef CONFIG_CGROUPS BTF_ID_FLAGS(func, bpf_cgroup_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) @@ -4885,9 +4887,9 @@ BTF_ID_FLAGS(func, bpf_rcu_read_lock) BTF_ID_FLAGS(func, bpf_rcu_read_unlock) BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL) BTF_ID_FLAGS(func, bpf_dynptr_slice_rdwr, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW) -BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY) +BTF_ID_FLAGS(func, bpf_iter_num_new, KF_ITER_NEW | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_iter_num_next, KF_ITER_NEXT | KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_iter_num_destroy, KF_ITER_DESTROY | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_iter_task_vma_new, KF_ITER_NEW | KF_RCU) BTF_ID_FLAGS(func, bpf_iter_task_vma_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_task_vma_destroy, KF_ITER_DESTROY) @@ -4962,8 +4964,8 @@ BTF_ID_FLAGS(func, bpf_strncasestr); #if defined(CONFIG_BPF_LSM) && defined(CONFIG_CGROUPS) BTF_ID_FLAGS(func, bpf_cgroup_read_xattr, KF_RCU) #endif -BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS) -BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_stream_vprintk, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_stream_print_stack, KF_IMPLICIT_ARGS | KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_task_work_schedule_signal, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_task_work_schedule_resume, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_dynptr_from_file) diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index e4e338cdb437..e527cb425cf4 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -744,10 +744,10 @@ __bpf_kfunc void bpf_res_spin_unlock_irqrestore(struct bpf_res_spin_lock *lock, __bpf_kfunc_end_defs(); BTF_KFUNCS_START(rqspinlock_kfunc_ids) -BTF_ID_FLAGS(func, bpf_res_spin_lock, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_res_spin_unlock) -BTF_ID_FLAGS(func, bpf_res_spin_lock_irqsave, KF_RET_NULL) -BTF_ID_FLAGS(func, bpf_res_spin_unlock_irqrestore) +BTF_ID_FLAGS(func, bpf_res_spin_lock, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_unlock, KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_lock_irqsave, KF_RET_NULL | KF_SPINLOCK_SAFE) +BTF_ID_FLAGS(func, bpf_res_spin_unlock_irqrestore, KF_SPINLOCK_SAFE) BTF_KFUNCS_END(rqspinlock_kfunc_ids) static const struct btf_kfunc_id_set rqspinlock_kfunc_set = { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d925197c2e5f..d952bd95cbb7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11802,20 +11802,6 @@ static bool is_bpf_rbtree_api_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_rbtree_right]; } -static bool is_bpf_iter_num_api_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_iter_num_new] || - btf_id == special_kfunc_list[KF_bpf_iter_num_next] || - btf_id == special_kfunc_list[KF_bpf_iter_num_destroy]; -} - -static bool is_bpf_graph_api_kfunc(u32 btf_id) -{ - return is_bpf_list_api_kfunc(btf_id) || - is_bpf_rbtree_api_kfunc(btf_id) || - is_bpf_refcount_acquire_kfunc(btf_id); -} - static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) { return btf_id == special_kfunc_list[KF_bpf_res_spin_lock] || @@ -11824,24 +11810,16 @@ static bool is_bpf_res_spin_lock_kfunc(u32 btf_id) btf_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore]; } -static bool is_bpf_arena_kfunc(u32 btf_id) +static bool kfunc_spin_allowed(struct bpf_verifier_env *env, s32 func_id, s16 offset) { - return btf_id == special_kfunc_list[KF_bpf_arena_alloc_pages] || - btf_id == special_kfunc_list[KF_bpf_arena_free_pages] || - btf_id == special_kfunc_list[KF_bpf_arena_reserve_pages]; -} + struct bpf_kfunc_meta kfunc; + int err; -static bool is_bpf_stream_kfunc(u32 btf_id) -{ - return btf_id == special_kfunc_list[KF_bpf_stream_vprintk] || - btf_id == special_kfunc_list[KF_bpf_stream_print_stack]; -} + err = fetch_kfunc_meta(env, func_id, offset, &kfunc); + if (err || !kfunc.flags) + return false; -static bool kfunc_spin_allowed(u32 btf_id) -{ - return is_bpf_graph_api_kfunc(btf_id) || is_bpf_iter_num_api_kfunc(btf_id) || - is_bpf_res_spin_lock_kfunc(btf_id) || is_bpf_arena_kfunc(btf_id) || - is_bpf_stream_kfunc(btf_id); + return *kfunc.flags & KF_SPINLOCK_SAFE; } static bool is_sync_callback_calling_kfunc(u32 btf_id) @@ -17420,7 +17398,7 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) insn->imm != BPF_FUNC_spin_unlock && insn->imm != BPF_FUNC_kptr_xchg) || (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && - (insn->off != 0 || !kfunc_spin_allowed(insn->imm)))) { + !kfunc_spin_allowed(env, insn->imm, insn->off))) { verbose(env, "function calls are not allowed while holding a lock\n"); return -EINVAL; diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c index 7af5560f2a08..2b39cc1b09f9 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c +++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c @@ -71,8 +71,10 @@ static struct kfunc_test_params kfunc_tests[] = { TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"), TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"), TC_FAIL(kfunc_call_test_pointer_arg_type_mismatch, 0, "R1 expected pointer to ctx, but got scalar"), + TC_FAIL(kfunc_call_test_spin_lock_unsafe, 0, "function calls are not allowed while holding a lock"), /* success cases */ + TC_TEST(kfunc_call_test_spin_lock_safe, 0), TC_TEST(kfunc_call_test1, 12), TC_TEST(kfunc_call_test2, 3), TC_TEST(kfunc_call_test4, -1234), diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c index 64b6a0b0ab1c..7e93f7fb1329 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c @@ -4,6 +4,18 @@ #include #include "../test_kmods/bpf_testmod_kfunc.h" +static struct bpf_spin_lock kfunc_call_lock SEC(".data.A"); + +SEC("?tc") +int kfunc_call_test_spin_lock_unsafe(struct __sk_buff *skb) +{ + bpf_spin_lock(&kfunc_call_lock); + bpf_kfunc_trigger_ctx_check(); + bpf_spin_unlock(&kfunc_call_lock); + + return 0; +} + struct syscall_test_args { __u8 data[16]; size_t size; diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_test.c b/tools/testing/selftests/bpf/progs/kfunc_call_test.c index 5edc51564f71..8e6560c31e78 100644 --- a/tools/testing/selftests/bpf/progs/kfunc_call_test.c +++ b/tools/testing/selftests/bpf/progs/kfunc_call_test.c @@ -5,6 +5,18 @@ #include "bpf_misc.h" #include "../test_kmods/bpf_testmod_kfunc.h" +static struct bpf_spin_lock kfunc_call_lock SEC(".data.A"); + +SEC("tc") +int kfunc_call_test_spin_lock_safe(struct __sk_buff *skb) +{ + bpf_spin_lock(&kfunc_call_lock); + bpf_testmod_test_mod_kfunc(42); + bpf_spin_unlock(&kfunc_call_lock); + + return 0; +} + SEC("tc") int kfunc_call_test5(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index eb0f9b5e18d8..0585794606ed 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -1384,7 +1384,7 @@ __bpf_kfunc void bpf_kfunc_trigger_ctx_check(void) } BTF_KFUNCS_START(bpf_testmod_check_kfunc_ids) -BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc) +BTF_ID_FLAGS(func, bpf_testmod_test_mod_kfunc, KF_SPINLOCK_SAFE) BTF_ID_FLAGS(func, bpf_kfunc_call_test1) BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3)