diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index cf2bf388e181..2fa6b8dae8c4 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -810,8 +810,14 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, vma = find_vma(mm, start); if (vma && vma->vm_start <= start && vma->vm_end > start) { + /* + * The callback must not sleep while mmap_lock is held, and the + * verifier checks it as an RCU read-side critical section. + */ + rcu_read_lock(); callback_fn((u64)(long)task, (u64)(long)vma, (u64)(long)callback_ctx, 0, 0); + rcu_read_unlock(); ret = 0; } bpf_mmap_unlock_mm(work, mm); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 883cf148235c..430d5309eb5c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11629,6 +11629,8 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env, bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; + /* bpf_find_vma() runs the callback under mmap_lock and RCU. */ + callee->in_rcu_callback_fn = true; callee->callback_ret_range = retval_range(0, 1); return 0; }