diff --git a/Documentation/bpf/bpf_iterators.rst b/Documentation/bpf/bpf_iterators.rst index 189e3ec1c6c8..72435e32ffa1 100644 --- a/Documentation/bpf/bpf_iterators.rst +++ b/Documentation/bpf/bpf_iterators.rst @@ -86,6 +86,16 @@ Constructor/next/destructor implementation contract is as follows: resources and marks stack space used by `struct bpf_iter_` as usable for something else. Destructor is marked with KF_ITER_DESTROY flag. +The verifier treats a non-RCU, non-acquired struct result as borrowed from +the iterator's current element. It invalidates the previous result and its +tracked derived pointers on the next call, including when it returns NULL, +and on destruction of the iterator. Resources borrowing that lifetime must +be released first. A next method whose struct result has an independent +lifetime must use ``KF_ITER_RET_INDEPENDENT``. This flag requires +``KF_ITER_NEXT`` and a struct pointer return type, and cannot be combined +with ``KF_ACQUIRE``. RCU-protected and acquired results retain their own +lifetime rules. + Any open-coded BPF iterator implementation has to implement at least these three methods. It is enforced that for any given type of iterator only applicable constructor/destructor/next are callable. I.e., verifier ensures diff --git a/include/linux/btf.h b/include/linux/btf.h index b66e20a18c54..a18254b7521b 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -82,6 +82,7 @@ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ #define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ #define KF_PERFMON (1 << 18) /* kfunc requires CAP_PERFMON */ +#define KF_ITER_RET_INDEPENDENT (1 << 19) /* next result outlives the current element */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 94d43612d211..7ed1ff71a089 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -9720,6 +9720,10 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, if (!flags || (flags & (flags - 1))) return -EINVAL; + if ((func_flags & KF_ITER_RET_INDEPENDENT) && + (!(flags & KF_ITER_NEXT) || (func_flags & KF_ACQUIRE))) + return -EINVAL; + /* any BPF iter kfunc should have `struct bpf_iter_ *` first arg */ nr_args = btf_type_vlen(func); if (nr_args < 1) @@ -9760,6 +9764,12 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, t = btf_type_skip_modifiers(btf, func->type, NULL); if (!t || !btf_type_is_ptr(t)) return -EINVAL; + /* The opt-out applies only to non-acquired struct results. */ + if (func_flags & KF_ITER_RET_INDEPENDENT) { + t = btf_type_skip_modifiers(btf, t->type, NULL); + if (!t || !__btf_type_is_struct(t)) + return -EINVAL; + } } if (flags & KF_ITER_DESTROY) { @@ -9826,7 +9836,8 @@ static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) if (!func || !btf_type_is_func_proto(func)) return -EINVAL; - if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY)) { + if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY | + KF_ITER_RET_INDEPENDENT)) { err = btf_check_iter_kfuncs(btf, func_name, func, func_flags); if (err) return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5601d2056218..c223bf3bc9cd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -211,6 +211,9 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par static int __release_reference_nomark(struct bpf_verifier_state *state, int id); static int release_reference_nomark(struct bpf_verifier_env *env, int id); static int release_reference(struct bpf_verifier_env *env, int id); +static int invalidate_reference(struct bpf_verifier_env *env, int id); +static u32 reg_lifetime_id(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); @@ -793,8 +796,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ if (err) return err; - /* Track parent's id if the parent is a referenced object */ - parent_id = ref_obj->id; + /* All non-clone constructors take their backing object in R1. */ + parent_id = reg_lifetime_id(env, &cur_regs(env)[BPF_REG_1]); if (dynptr_type_referenced(type)) { int id; @@ -1074,7 +1077,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) { struct bpf_func_state *state = bpf_func(env, reg); - int spi, i, j; + int spi, i, j, err; spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) @@ -1084,8 +1087,11 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; - if (i == 0) - WARN_ON_ONCE(release_reference(env, st->id)); + if (i == 0) { + err = release_reference(env, st->id); + if (err) + return err; + } bpf_mark_reg_not_init(env, st); @@ -1627,6 +1633,12 @@ static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg return find_reference_state(env->cur_state, reg->id); } +static u32 reg_lifetime_id(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + return reg_is_referenced(env, reg) ? reg->id : reg->parent_id; +} + static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) { struct bpf_verifier_state *state = env->cur_state; @@ -6848,9 +6860,14 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, flag &= ~PTR_MAYBE_NULL; if (atype == BPF_READ && value_regno >= 0) { + u32 parent_id = reg_lifetime_id(env, reg); + ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) return ret; + if ((regs[value_regno].type & PTR_TRUSTED) && + !(regs[value_regno].type & (MEM_RCU | MEM_PERCPU | MEM_USER))) + regs[value_regno].parent_id = parent_id; } return 0; @@ -8636,6 +8653,21 @@ static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_ITER_DESTROY; } +static bool iter_next_borrows_struct(struct bpf_call_arg_meta *meta) +{ + const struct btf_type *t; + + if (!(meta->kfunc_flags & KF_ITER_NEXT) || + (meta->kfunc_flags & (KF_ITER_RET_INDEPENDENT | KF_ACQUIRE))) + return false; + + t = btf_type_skip_modifiers(meta->btf, meta->func_proto->type, NULL); + if (!t || !btf_type_is_ptr(t)) + return false; + t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + return t && __btf_type_is_struct(t); +} + static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, const struct btf_param *arg) { @@ -8746,6 +8778,13 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * meta->iter.frameno = reg->frameno; update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); + /* The iterator stays. What was borrowed from the previous element goes. */ + if (iter_next_borrows_struct(meta)) { + err = invalidate_reference(env, meta->ref_obj.id); + if (err) + return err; + } + if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); if (err) @@ -11073,8 +11112,8 @@ static int idstack_pop(struct bpf_idmap *idmap) return idmap->map[--idmap->cnt].old; } -/* Release id and objects derived from it iteratively in a DFS manner */ -static int release_reference(struct bpf_verifier_env *env, int id) +/* Invalidate id and objects derived from it iteratively in a DFS manner */ +static int invalidate_reference(struct bpf_verifier_env *env, int id) { u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR); struct bpf_verifier_state *vstate = env->cur_state; @@ -11089,11 +11128,6 @@ static int release_reference(struct bpf_verifier_env *env, int id) if (err) return err; - if (find_reference_state(vstate, id)) { - err = release_reference_nomark(env, id); - WARN_ON_ONCE(err); - } - while ((id = idstack_pop(idstack))) { /* * Child references are inaccessible after parent is released, @@ -11145,6 +11179,14 @@ static int release_reference(struct bpf_verifier_env *env, int id) return 0; } +static int release_reference(struct bpf_verifier_env *env, int id) +{ + if (find_reference_state(env->cur_state, id)) + WARN_ON_ONCE(release_reference_nomark(env, id)); + + return invalidate_reference(env, id); +} + static void invalidate_non_owning_refs(struct bpf_verifier_env *env) { struct bpf_func_state *unused; @@ -15822,6 +15864,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].type = type; regs[BPF_REG_0].btf_id = ptr_type_id; + if (iter_next_borrows_struct(&meta) && !(type & MEM_RCU)) + regs[BPF_REG_0].parent_id = meta.ref_obj.id; } if (is_kfunc_ret_null(&meta)) {