github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-30 18:13:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: use rcu_read_lock_dont_migrate() for trampoline.c

Browse Source 
Use rcu_read_lock_dont_migrate() and rcu_read_unlock_migrate() in
trampoline.c to obtain better performance when PREEMPT_RCU is not enabled.

Signed-off-by: Menglong Dong <dongml2@chinatelecom.cn>
Link: https://lore.kernel.org/r/20250821090609.42508-8-dongml2@chinatelecom.cn
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Menglong Dong 2025-08-21 17:06:09 +08:00 committed by Alexei Starovoitov
parent 427a36bb55
commit 8e4f0b1ebc
1 changed files with 6 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
kernel/bpf/trampoline.c
View File

@ -899,8 +899,7 @@ static __always_inline u64 notrace bpf_prog_start_time(void)
static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx) static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
__acquires(RCU) __acquires(RCU)
{ {
rcu_read_lock(); rcu_read_lock_dont_migrate();
migrate_disable();
run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
@ -949,8 +948,7 @@ static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,
update_prog_stats(prog, start); update_prog_stats(prog, start);
this_cpu_dec(*(prog->active)); this_cpu_dec(*(prog->active));
migrate_enable(); rcu_read_unlock_migrate();
rcu_read_unlock();
} }
static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog, static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
@ -960,8 +958,7 @@ static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
/* Runtime stats are exported via actual BPF_LSM_CGROUP /* Runtime stats are exported via actual BPF_LSM_CGROUP
* programs, not the shims. * programs, not the shims.
*/ */
rcu_read_lock(); rcu_read_lock_dont_migrate();
migrate_disable();
run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
@ -974,8 +971,7 @@ static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
{ {
bpf_reset_run_ctx(run_ctx->saved_run_ctx); bpf_reset_run_ctx(run_ctx->saved_run_ctx);
migrate_enable(); rcu_read_unlock_migrate();
rcu_read_unlock();
} }
u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,
@ -1033,8 +1029,7 @@ static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,
struct bpf_tramp_run_ctx *run_ctx) struct bpf_tramp_run_ctx *run_ctx)
__acquires(RCU) __acquires(RCU)
{ {
rcu_read_lock(); rcu_read_lock_dont_migrate();
migrate_disable();
run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx); run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
@ -1048,8 +1043,7 @@ static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,
bpf_reset_run_ctx(run_ctx->saved_run_ctx); bpf_reset_run_ctx(run_ctx->saved_run_ctx);
update_prog_stats(prog, start); update_prog_stats(prog, start);
migrate_enable(); rcu_read_unlock_migrate();
rcu_read_unlock();
} }
void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr) void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)