diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 112ca1511da4..816948df8fff 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -379,6 +379,11 @@ struct bpf_func_state { bool in_async_callback_fn; bool in_exception_callback_fn; bool no_stack_arg_load; + /* + * The helper invokes this callback under rcu_read_lock(), so the + * callback and its callees run in an RCU read-side critical section. + */ + bool in_rcu_callback_fn; /* For callback calling functions that limit number of possible * callback executions (e.g. bpf_loop) keeps track of current * simulated iteration number. diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 953700822fa7..b3d250caaaa6 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -995,6 +995,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat if (!old->no_stack_arg_load && cur->no_stack_arg_load) return false; + if (old->in_rcu_callback_fn != cur->in_rcu_callback_fn) + return false; + for (i = 0; i < MAX_BPF_REG; i++) if (((1 << i) & live_regs) && !regsafe(env, &old->regs[i], &cur->regs[i], diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ddfa0c6d2755..883cf148235c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11530,6 +11530,19 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env, if (err) return err; + switch (map->map_type) { + case BPF_MAP_TYPE_HASH: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_RHASH: + /* Hash tables run the callback under rcu_read_lock(). */ + callee->in_rcu_callback_fn = true; + break; + default: + break; + } + callee->in_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; @@ -12273,10 +12286,22 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, return *ptr && (*ptr)->func ? 0 : -EINVAL; } +/* Is any frame on the current path a callback that runs under RCU? */ +static bool in_rcu_callback(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int i; + + for (i = 0; i <= state->curframe; i++) + if (state->frame[i]->in_rcu_callback_fn) + return true; + return false; +} + /* Check if we're in a sleepable context. */ static inline bool in_sleepable_context(struct bpf_verifier_env *env) { - return !in_rcu_cs(env); + return !in_rcu_cs(env) && !in_rcu_callback(env); } static const char *non_sleepable_context_description(struct bpf_verifier_env *env) @@ -12290,6 +12315,7 @@ static const char *non_sleepable_context_description(struct bpf_verifier_env *en return "IRQ-disabled region"; if (env->cur_state->active_locks) return "lock region"; + return "callback running under rcu_read_lock"; } return "non-sleepable prog"; }