diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 7c376451db82..27b43fda9b17 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -946,6 +946,7 @@ struct bpf_verifier_env { bool seen_direct_write; bool seen_exception; bool signature; + u32 insn_aux_data_len; struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; diff --git a/include/linux/filter.h b/include/linux/filter.h index 15d83684c6e9..4a9bc6a848f2 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1267,25 +1267,12 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, #ifdef CONFIG_BPF_SYSCALL struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len); -struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env); -void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux); #else static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch, u32 len) { return ERR_PTR(-ENOTSUPP); } - -static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) -{ - return NULL; -} - -static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux) -{ -} #endif /* CONFIG_BPF_SYSCALL */ int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index a3e1fae32eac..6a94370a2448 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2634,22 +2634,10 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc { #ifdef CONFIG_BPF_JIT struct bpf_prog *orig_prog; - struct bpf_insn_aux_data *orig_insn_aux; if (!bpf_prog_need_blind(prog)) return bpf_int_jit_compile(env, prog); - if (env) { - /* - * If env is not NULL, we are called from the end of bpf_check(), at this - * point, only insn_aux_data is used after failure, so it should be restored - * on failure. - */ - orig_insn_aux = bpf_dup_insn_aux_data(env); - if (!orig_insn_aux) - return prog; - } - orig_prog = prog; prog = bpf_jit_blind_constants(env, prog); /* @@ -2662,8 +2650,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc prog = bpf_int_jit_compile(env, prog); if (prog->jited) { bpf_jit_prog_release_other(prog, orig_prog); - if (env) - vfree(orig_insn_aux); return prog; } @@ -2671,8 +2657,6 @@ static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struc out_restore: prog = orig_prog; - if (env) - bpf_restore_insn_aux_data(env, orig_insn_aux); #endif return prog; } diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index c4bd70befbb5..2417a3461652 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -230,6 +230,7 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, if (cnt == 1) return; prog_len = new_prog->len; + env->insn_aux_data_len = prog_len; memmove(data + off + cnt - 1, data + off, sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); @@ -496,7 +497,6 @@ static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) { struct bpf_insn_aux_data *aux_data = env->insn_aux_data; - struct bpf_insn *insns = env->prog->insnsi; int end = start + len; int i; @@ -505,9 +505,6 @@ void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len) kvfree(aux_data[i].jt); aux_data[i].jt = NULL; } - - if (bpf_is_ldimm64(&insns[i])) - i++; } } @@ -520,7 +517,6 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) if (bpf_prog_is_offloaded(env->prog->aux)) bpf_prog_offload_remove_insns(env, off, cnt); - /* Should be called before bpf_remove_insns, as it uses prog->insnsi */ bpf_clear_insn_aux_data(env, off, cnt); err = bpf_remove_insns(env->prog, off, cnt); @@ -539,6 +535,7 @@ static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) memmove(aux_data + off, aux_data + off + cnt, sizeof(*aux_data) * (orig_prog_len - off - cnt)); + env->insn_aux_data_len -= cnt; return 0; } @@ -1057,26 +1054,6 @@ static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_s env->subprog_info[env->subprog_cnt].start = env->prog->len; } -struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env) -{ - size_t size; - void *new_aux; - - size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len); - new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT); - if (new_aux) - memcpy(new_aux, env->insn_aux_data, size); - return new_aux; -} - -void bpf_restore_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_insn_aux_data *orig_insn_aux) -{ - /* the expanded elements are zero-filled, so no special handling is required */ - vfree(env->insn_aux_data); - env->insn_aux_data = orig_insn_aux; -} - static int jit_subprogs(struct bpf_verifier_env *env) { struct bpf_prog *prog = env->prog, **func, *tmp; @@ -1351,7 +1328,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) bool blinded = false; struct bpf_insn *insn; struct bpf_prog *prog, *orig_prog; - struct bpf_insn_aux_data *orig_insn_aux; u32 *orig_subprog_starts; if (env->subprog_cnt <= 1) @@ -1359,14 +1335,8 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) prog = orig_prog = env->prog; if (bpf_prog_need_blind(prog)) { - orig_insn_aux = bpf_dup_insn_aux_data(env); - if (!orig_insn_aux) { - err = -ENOMEM; - goto out_cleanup; - } orig_subprog_starts = bpf_dup_subprog_starts(env); if (!orig_subprog_starts) { - vfree(orig_insn_aux); err = -ENOMEM; goto out_cleanup; } @@ -1386,7 +1356,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) if (blinded) { bpf_jit_prog_release_other(prog, orig_prog); kvfree(orig_subprog_starts); - vfree(orig_insn_aux); } return 0; @@ -1416,7 +1385,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) out_restore: bpf_restore_subprog_starts(env, orig_subprog_starts); - bpf_restore_insn_aux_data(env, orig_insn_aux); kvfree(orig_subprog_starts); out_cleanup: /* cleanup main prog to be interpreted */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 73d6cd563cdf..7a4e1b88d73a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20213,7 +20213,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (!is_priv) mutex_lock(&bpf_verifier_lock); - len = env->prog->len; + len = env->insn_aux_data_len = env->prog->len; env->insn_aux_data = __vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len), GFP_KERNEL_ACCOUNT | __GFP_ZERO); @@ -20468,7 +20468,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, release_btfs(env); err_free_env: if (env->insn_aux_data) - bpf_clear_insn_aux_data(env, 0, env->prog->len); + bpf_clear_insn_aux_data(env, 0, env->insn_aux_data_len); vfree(env->insn_aux_data); kvfree(env->fd_array); bpf_stack_liveness_free(env);