diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d952bd95cbb7..e6233c0081d1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4452,7 +4452,9 @@ static bool in_sleepable(struct bpf_verifier_env *env) static bool in_rcu_cs(struct bpf_verifier_env *env) { return env->cur_state->active_rcu_locks || + env->cur_state->active_preempt_locks || env->cur_state->active_locks || + env->cur_state->active_irq_id || !in_sleepable(env); } @@ -7166,7 +7168,6 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state return err; } } else { - bool was_in_rcu_cs; void *ptr; int type; @@ -7194,12 +7195,11 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state verbose(env, "%s_unlock cannot be out of order\n", lock_str); return -EINVAL; } - was_in_rcu_cs = in_rcu_cs(env); if (release_lock_state(cur, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); return -EINVAL; } - if (was_in_rcu_cs && !in_rcu_cs(env)) + if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); invalidate_non_owning_refs(env); @@ -11663,6 +11663,9 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * err = unmark_stack_slot_irq_flag(env, reg, kfunc_class); if (err) return err; + + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); } return 0; } @@ -13159,7 +13162,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); return -EINVAL; } - if (--env->cur_state->active_rcu_locks == 0) + env->cur_state->active_rcu_locks--; + if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); } else if (preempt_disable) { env->cur_state->active_preempt_locks++; @@ -13169,6 +13173,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EINVAL; } env->cur_state->active_preempt_locks--; + if (!in_rcu_cs(env)) + invalidate_rcu_protected_refs(env); } if (sleepable && !in_sleepable_context(env)) { diff --git a/tools/testing/selftests/bpf/progs/cpumask_failure.c b/tools/testing/selftests/bpf/progs/cpumask_failure.c index 74b4cd4bcdbb..4628feb53d86 100644 --- a/tools/testing/selftests/bpf/progs/cpumask_failure.c +++ b/tools/testing/selftests/bpf/progs/cpumask_failure.c @@ -116,9 +116,9 @@ int BPF_PROG(test_cpumask_null, struct task_struct *task, u64 clone_flags) return 0; } -SEC("tp_btf/task_newtask") +SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("R2 must be a rcu pointer") -int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_flags) +int BPF_PROG(test_global_mask_out_of_rcu) { struct bpf_cpumask *local, *prev; @@ -133,6 +133,10 @@ int BPF_PROG(test_global_mask_out_of_rcu, struct task_struct *task, u64 clone_fl return 0; } + /* + * Use a sleepable program so explicit RCU is the only source of RCU + * protection. + */ bpf_rcu_read_lock(); local = global_mask; if (!local) {