diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 54144372281c..46d325aa178e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3137,7 +3137,8 @@ typedef const struct bpf_func_proto * const struct bpf_prog *prog); enum bpf_iter_feature { - BPF_ITER_RESCHED = BIT(0), + BPF_ITER_RESCHED = BIT(0), /* may cond_resched() between objects */ + BPF_ITER_NOSLEEP = BIT(1), /* ->show() can't run sleepable progs */ }; #define BPF_ITER_CTX_ARG_MAX 2 diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index b40eb404adab..fc3a75684ef7 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -72,6 +72,15 @@ static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_ return tinfo->reg_info->feature & BPF_ITER_RESCHED; } +/* Only allow sleepable program for resched-able iterator that can run one */ +static bool bpf_iter_target_support_prog(const struct bpf_iter_target_info *tinfo, + const struct bpf_prog *prog) +{ + return !prog->sleepable || + (bpf_iter_target_support_resched(tinfo) && + !(tinfo->reg_info->feature & BPF_ITER_NOSLEEP)); +} + static bool bpf_iter_support_resched(struct seq_file *seq) { struct bpf_iter_priv_data *iter_priv; @@ -409,6 +418,8 @@ static int bpf_iter_link_replace(struct bpf_link *link, struct bpf_prog *new_prog, struct bpf_prog *old_prog) { + struct bpf_iter_link *iter_link = + container_of(link, struct bpf_iter_link, link); int ret = 0; mutex_lock(&link_mutex); @@ -419,7 +430,8 @@ static int bpf_iter_link_replace(struct bpf_link *link, if (link->prog->type != new_prog->type || link->prog->expected_attach_type != new_prog->expected_attach_type || - link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { + link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id || + !bpf_iter_target_support_prog(iter_link->tinfo, new_prog)) { ret = -EINVAL; goto out_unlock; } @@ -544,8 +556,7 @@ int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, if (!tinfo) return -ENOENT; - /* Only allow sleepable program for resched-able iterator */ - if (prog->sleepable && !bpf_iter_target_support_resched(tinfo)) + if (!bpf_iter_target_support_prog(tinfo, prog)) return -EINVAL; link = kzalloc_obj(*link, GFP_USER | __GFP_NOWARN); diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index 13e1aabe6f88..cf2bf388e181 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -737,7 +737,11 @@ static const struct bpf_iter_seq_info task_vma_seq_info = { static struct bpf_iter_reg task_vma_reg_info = { .target = "task_vma", .attach_target = bpf_iter_attach_task, - .feature = BPF_ITER_RESCHED, + /* + * ->show() runs with the mmap_lock of the iterated mm held, which a + * sleepable program takes again when it faults on user memory. + */ + .feature = BPF_ITER_RESCHED | BPF_ITER_NOSLEEP, .ctx_arg_info_size = 2, .ctx_arg_info = { { offsetof(struct bpf_iter__task_vma, task),