diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 04cadd987169..ffa5626411ac 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4147,8 +4147,16 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off); -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump); +struct bpf_linfo_source { + const char *file; + const char *line; + u32 file_name_off; + int line_num; + int line_col; +}; + +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src); int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump); struct bpf_prog *bpf_prog_find_from_stack(void); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 27b43fda9b17..bc2af02547fe 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -354,6 +354,11 @@ struct bpf_func_state { * 0 = main function, 1 = first callee. */ u32 frameno; + /* + * Unique diagnostic identity for this function invocation. Frame depth is + * reused after returns, while this ID is preserved across state clones. + */ + u32 diag_frame_id; /* subprog number == index within subprog_info * zero == main subprog */ @@ -833,6 +838,7 @@ static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) return 0; } +struct bpf_diag; struct bpf_verifier_env; struct backtrack_state { @@ -950,6 +956,7 @@ struct bpf_verifier_env { struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; + struct bpf_diag *diag; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; @@ -1349,6 +1356,18 @@ static inline bool type_is_non_owning_ref(u32 type) return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; } +static inline bool type_is_map_ptr(enum bpf_reg_type type) +{ + switch (base_type(type)) { + case CONST_PTR_TO_MAP: + case PTR_TO_MAP_KEY: + case PTR_TO_MAP_VALUE: + return true; + default: + return false; + } +} + static inline bool type_is_pkt_pointer(enum bpf_reg_type type) { type = base_type(type); @@ -1433,8 +1452,10 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, u32 frameno); u32 bpf_vlog_alignment(u32 pos); +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn); struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off); +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); diff --git a/include/linux/btf.h b/include/linux/btf.h index 3f5255d095a2..7ea13768c979 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -214,6 +214,7 @@ int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, void *obj, */ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, char *buf, int len, u64 flags); +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len); int btf_get_fd_by_id(u32 id); u32 btf_obj_id(const struct btf *btf); diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 4dc41bf5780c..90255d80e5be 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse endif CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o diagnostics.o obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 87ffde865a50..5b9d767895c9 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8316,6 +8316,16 @@ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, return ssnprintf.len; } +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len) +{ + struct btf_show show = { + .btf = btf, + .state.type_id = type_id, + }; + + return snprintf(buf, len, "%s", btf_show_name(&show)); +} + #ifdef CONFIG_PROC_FS static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) { diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index 818f7afac83a..0f13c13f4133 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -5,6 +5,8 @@ #include #include +#include "diagnostics.h" + #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args) /* non-recursive DFS pseudo code @@ -112,6 +114,10 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) if (w < 0 || w >= env->prog->len) { verbose_linfo(env, t, "%d: ", t); verbose(env, "jump out of range from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "jump out of range", "Keep branch targets inside the program.", + "Instruction %d jumps to instruction %d, but the program only contains instructions 0 through %d.", + t, w, env->prog->len - 1); return -EINVAL; } @@ -135,6 +141,11 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) verbose_linfo(env, t, "%d: ", t); verbose_linfo(env, w, "%d: ", w); verbose(env, "back-edge from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "back-edge is not allowed", + "Load with privileges that allow this back-edge, or rewrite the control flow so it does not branch backward.", + "Instruction %d branches back to instruction %d. This program is being rejected without the privilege needed for this back-edge.", + t, w); return -EINVAL; } else if (insn_state[w] == EXPLORED) { /* forward- or cross-edge */ @@ -315,6 +326,11 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, if (!jt) { verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start); + bpf_diag_program_structure( + env, subprog_start, "missing jump table", + "Make sure subprograms containing gotox instructions are accompanied by jump tables referencing these subprograms.", + "No jump table was found for the subprogram that starts at instruction %u.", + subprog_start); return ERR_PTR(-EINVAL); } @@ -342,6 +358,11 @@ create_jt(int t, struct bpf_verifier_env *env) if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) { verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n", t, subprog_start, subprog_end); + bpf_diag_program_structure( + env, t, "jump table target out of range", + "Keep every jump-table target inside the same subprogram.", + "The jump table for instruction %d points outside subprogram range [%u,%u).", + t, subprog_start, subprog_end); kvfree(jt); return ERR_PTR(-EINVAL); } @@ -373,6 +394,11 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) w = jt->items[i]; if (w < 0 || w >= env->prog->len) { verbose(env, "indirect jump out of range from insn %d to %d\n", t, w); + bpf_diag_program_structure( + env, t, "indirect jump out of range", + "Keep indirect jump targets inside the program.", + "Instruction %d can jump indirectly to instruction %d, but the program only contains instructions 0 through %d.", + t, w, env->prog->len - 1); return -EINVAL; } @@ -623,12 +649,21 @@ int bpf_check_cfg(struct bpf_verifier_env *env) if (insn_state[i] != EXPLORED) { verbose(env, "unreachable insn %d\n", i); + bpf_diag_program_structure( + env, i, "unreachable instruction", + "Remove the unreachable instruction or add valid control flow that reaches it.", + "Instruction %d is not reachable from the program entry point.", i); ret = -EINVAL; goto err_free; } if (bpf_is_ldimm64(insn)) { if (insn_state[i + 1] != 0) { verbose(env, "jump into the middle of ldimm64 insn %d\n", i); + bpf_diag_program_structure( + env, i, "jump into ldimm64 immediate", + "Target the first instruction of the ldimm64 pair, or restructure the jump target.", + "Control flow reaches the second half of the ldimm64 instruction pair that starts at instruction %d.", + i); ret = -EINVAL; goto err_free; } diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6a94370a2448..d55e737ed75a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3461,24 +3461,14 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); #ifdef CONFIG_BPF_SYSCALL -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump) +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src) { - /* Get base component of the file path. */ - if (filep) { - *filep = btf_name_by_offset(btf, linfo->file_name_off); - *filep = kbasename(*filep); - } - - /* Obtain the source line, and strip whitespace in prefix. */ - if (linep) { - *linep = btf_name_by_offset(btf, linfo->line_off); - while (isspace(**linep)) - *linep += 1; - } - - if (nump) - *nump = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->file = kbasename(btf_name_by_offset(btf, linfo->file_name_off)); + src->line = btf_name_by_offset(btf, linfo->line_off); + src->file_name_off = linfo->file_name_off; + src->line_num = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->line_col = BPF_LINE_INFO_LINE_COL(linfo->line_col); } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off) @@ -3521,6 +3511,7 @@ const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump) { + struct bpf_linfo_source src; int idx = -1, insn_start, insn_end, len; struct bpf_line_info *linfo; void **jited_linfo; @@ -3552,7 +3543,15 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * if (idx == -1) return -ENOENT; - bpf_get_linfo_file_line(btf, &linfo[idx], filep, linep, nump); + bpf_get_linfo_source(btf, &linfo[idx], &src); + while (isspace(*src.line)) + src.line++; + if (filep) + *filep = src.file; + if (linep) + *linep = src.line; + if (nump) + *nump = src.line_num; return 0; } diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c new file mode 100644 index 000000000000..33b7d9e8e2c3 --- /dev/null +++ b/kernel/bpf/diagnostics.c @@ -0,0 +1,2355 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "disasm.h" +#include "diagnostics.h" + +#define REGISTER_TYPE_SAFETY "Register Type Safety" +#define MEMORY_SAFETY "Memory Safety" +#define RESOURCE_LIFETIME_SAFETY "Resource Lifetime Safety" +#define CALL_TYPE_SAFETY "Call Type Safety" +#define EXECUTION_CONTEXT_SAFETY "Execution Context Safety" +#define PROGRAM_STRUCTURE "Program Structure" +#define POLICY "Policy" + +#define BPF_DIAG_TEXT_WIDTH 100 +#define BPF_DIAG_TEXT_INDENT " " +#define BPF_DIAG_CONTEXT 2 +#define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) +#define BPF_DIAG_HISTORY_RENDER_MAX 64 +#define BPF_DIAG_SOURCE_LANE_WIDTH 88 +#define BPF_DIAG_TAB_WIDTH 8 +#define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) +#define BPF_DIAG_FMT_BUF_SIZE 256 +#define BPF_DIAG_EVENT_LOG_MAX_SIZE (64U << 20) +#define DISASM_LINE_LEN 160 + +enum bpf_diag_mod_target_kind { + BPF_DIAG_MOD_TARGET_NONE, + BPF_DIAG_MOD_TARGET_REG, + BPF_DIAG_MOD_TARGET_STACK_ARG, + BPF_DIAG_MOD_TARGET_STACK_SLOT, + BPF_DIAG_MOD_TARGET_STACK_RANGE, +}; + +struct bpf_diag_mod_target { + u32 frame_id; + union { + struct { + s16 min_off; + s16 max_off; + } range; + u16 spi; + u8 regno; + u8 stack_arg; + }; + u8 frameno; + u8 kind; +}; + +static struct bpf_diag_mod_target diag_reg_target(u32 frame_id, u8 frameno, u8 regno) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_REG, + .regno = regno, + }; +} + +static struct bpf_diag_mod_target diag_stack_arg_target(u32 frame_id, u8 frameno, u8 slot) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_ARG, + .stack_arg = slot, + }; +} + +static struct bpf_diag_mod_target diag_stack_slot_target(u32 frame_id, u8 frameno, u16 spi) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_SLOT, + .spi = spi, + }; +} + +static struct bpf_diag_mod_target diag_stack_range_target(u32 frame_id, u8 frameno, + s16 min_off, s16 max_off) +{ + return (struct bpf_diag_mod_target){ + .frame_id = frame_id, + .frameno = frameno, + .kind = BPF_DIAG_MOD_TARGET_STACK_RANGE, + .range.min_off = min_off, + .range.max_off = max_off, + }; +} + +struct bpf_diag_reg_snapshot { + u32 type; + u32 btf_id; + const struct bpf_map *map_ptr; + const struct btf *btf; + struct tnum var_off; + struct cnum64 r64; +}; + +enum bpf_diag_history_kind { + BPF_DIAG_HISTORY_BRANCH, + BPF_DIAG_HISTORY_MOD, + BPF_DIAG_HISTORY_REF_ACQUIRE, + BPF_DIAG_HISTORY_REF_RELEASE, + BPF_DIAG_HISTORY_CONTEXT, +}; + +struct bpf_diag_history_event { + u32 insn_idx : 24; + u32 kind : 8; + u8 in_lineage : 1; + union { + struct { + bool cond_true; + } branch; + struct { + struct bpf_diag_mod_target target; + struct bpf_diag_mod_target origin; + struct bpf_diag_reg_snapshot old, new; + u8 reason; + bool origin_valid; + } mod; + struct { + u32 ref_id; + } ref; + struct { + u32 depth; + u8 kind; + bool enter; + } ctx; + }; +}; + +enum bpf_diag_history_scope { + BPF_DIAG_HISTORY_SCOPE_REG, + BPF_DIAG_HISTORY_SCOPE_STACK_ARG, + BPF_DIAG_HISTORY_SCOPE_REF, + BPF_DIAG_HISTORY_SCOPE_CONTEXT, +}; + +struct bpf_diag_history_opts { + enum bpf_diag_history_scope scope; + u32 frame_id; + u32 frameno; + int regno; + int stack_arg_slot; + u32 ref_id; + enum bpf_diag_context_kind ctx_kind; + u32 ctx_depth; +}; + +static void diag_print_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_opts *opts); +static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, + const struct bpf_diag_mod_target *target); +static const char *diag_context_name(enum bpf_diag_context_kind kind); +struct disasm_line { + char text[DISASM_LINE_LEN]; + int idx; + bool valid; +}; + +struct disasm_ctx { + struct bpf_verifier_env *env; + struct seq_buf seq; +}; + +struct diag_fmt_chunk { + struct list_head node; + struct seq_buf seq; + char data[]; +}; + +struct diag_fmt_mark { + struct diag_fmt_chunk *chunk; + size_t len; +}; + +struct bpf_diag_log { + struct bpf_diag_history_event *events; + /* Sequence number of the oldest retained event on the active path. */ + u64 first_seq; + u32 cnt; + u32 cap; + u32 head; + bool growth_failed; +}; + +struct bpf_diag_scratch { + struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; + struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; +}; + +struct bpf_diag_mod_scope { + struct bpf_reg_state target_reg_snapshot; + struct bpf_diag_mod_target target; + struct bpf_diag_mod_target origin; + enum bpf_diag_mod_reason reason; + u32 insn_idx; + bool active; + bool origin_valid; +}; + +struct bpf_diag { + struct bpf_diag_log log; + struct bpf_diag_scratch scratch; + struct list_head fmt_chunks; + struct bpf_diag_mod_scope mod; + u32 frame_id_gen; +}; + +bool bpf_diag_enabled(const struct bpf_verifier_env *env) +{ + return env->log.level & BPF_LOG_LEVEL; +} + +static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); + +int bpf_diag_init(struct bpf_verifier_env *env) +{ + if (!bpf_diag_enabled(env)) + return 0; + + env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT); + if (!env->diag) + return -ENOMEM; + + INIT_LIST_HEAD(&env->diag->fmt_chunks); + return 0; +} + +void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state) +{ + if (env->diag) + state->diag_frame_id = ++env->diag->frame_id_gen; +} + +static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + size_t capacity, available; + char *buf; + + if (!diag || !size || size > INT_MAX) + return NULL; + + if (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (available >= size) + goto commit; + } + + capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); + chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); + if (!chunk) + return NULL; + + seq_buf_init(&chunk->seq, chunk->data, capacity); + list_add_tail(&chunk->node, &diag->fmt_chunks); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (WARN_ON_ONCE(available < size)) + return NULL; + +commit: + seq_buf_commit(&chunk->seq, size); + return buf; +} + +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size) +{ + char *buf; + + buf = diag_fmt_alloc(env, size); + if (buf) + buf[0] = '\0'; + return buf; +} + +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + va_list copy; + char *buf; + int len; + + va_copy(copy, args); + len = vsnprintf(NULL, 0, fmt, copy); + va_end(copy); + if (len < 0 || len == INT_MAX) + return ""; + + buf = diag_fmt_alloc(env, len + 1); + if (buf) + vsnprintf(buf, len + 1, fmt, args); + return buf ?: ""; +} + +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) +{ + const char *buf; + va_list args; + + va_start(args, fmt); + buf = bpf_diag_vfmt(env, fmt, args); + va_end(args); + return buf; +} + +static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_mark mark = {}; + + if (!diag || list_empty(&diag->fmt_chunks)) + return mark; + + mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + mark.len = mark.chunk->seq.len; + return mark; +} + +static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + + if (!diag) + return; + + while (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + if (chunk == mark.chunk) + break; + list_del(&chunk->node); + kfree(chunk); + } + + if (mark.chunk) { + mark.chunk->seq.len = mark.len; + seq_buf_str(&mark.chunk->seq); + } +} + +void bpf_diag_free(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + + diag_fmt_restore(env, (struct diag_fmt_mark){}); + kvfree(diag->log.events); + kfree(diag); + env->diag = NULL; +} + +static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + va_start(args, fmt); + bpf_verifier_vlog(&env->log, fmt, args); + va_end(args); +} + +static u64 log_end(const struct bpf_diag_log *log) +{ + return log->first_seq + log->cnt; +} + +static u32 log_pos(const struct bpf_diag_log *log, u32 idx) +{ + u32 pos = log->head + idx; + + return pos < log->cap ? pos : pos - log->cap; +} + +u64 bpf_diag_event_log_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + return diag ? log_end(&diag->log) : 0; +} + +void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos) +{ + struct bpf_diag *diag = env->diag; + struct bpf_diag_log *log; + u64 end_seq; + + if (!diag) + return; + + log = &diag->log; + end_seq = log_end(log); + if (WARN_ON_ONCE(log_pos > end_seq)) + log_pos = end_seq; + + /* + * A deep abandoned path may have rotated away the shared prefix. In + * that case, restart with an empty retained suffix and remember that + * every event before the restored mark is unavailable. + */ + if (log_pos <= log->first_seq) { + log->first_seq = log_pos; + log->head = 0; + log->cnt = 0; + return; + } + + log->cnt = log_pos - log->first_seq; +} + +u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state) +{ + u32 depth = 0; + int i; + + for (i = 0; i < state->acquired_refs; i++) { + if (state->refs[i].type == REF_TYPE_IRQ) + depth++; + } + + return depth; +} + +static void diag_append_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + struct bpf_diag_history_event *events; + struct bpf_diag *diag = env->diag; + struct bpf_diag_log *log; + u32 cap, max_events; + + if (!diag) + return; + log = &diag->log; + + if (log->cnt < log->cap) { + log->events[log_pos(log, log->cnt++)] = *event; + return; + } + + max_events = BPF_DIAG_EVENT_LOG_MAX_SIZE / sizeof(*events); + if (log->growth_failed || log->cap == max_events) + goto rotate; + + cap = min(log->cap ? log->cap * 2 : 64, max_events); + events = kvrealloc(log->events, array_size(cap, sizeof(*events)), GFP_KERNEL_ACCOUNT); + if (!events) { + log->growth_failed = true; + goto rotate; + } + log->events = events; + log->cap = cap; + log->events[log->cnt++] = *event; + return; + +rotate: + if (log->cap) { + log->events[log->head++] = *event; + if (log->head == log->cap) + log->head = 0; + } + log->first_seq++; +} + +static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, + const char *next_prefix, const char *text) +{ + const char *prefix = first_prefix; + + while (*text) { + const char *line = text; + int prefix_len = strlen(prefix); + int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len; + int len = 0, last_space = -1; + + if (text_width < 1) + text_width = 1; + + while (line[len] && line[len] != '\n' && len < text_width) { + if (line[len] == ' ') + last_space = len; + len++; + } + + if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0) + len = last_space; + + diag_write(env, "%s%.*s\n", prefix, len, line); + + text = line + len; + while (*text == ' ') + text++; + if (*text == '\n') + text++; + + prefix = next_prefix; + } +} + +const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id) +{ + char *buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + size_t len; + int ret; + + if (!buf) + return ""; + + buf[0] = '\0'; + ret = btf_type_name_to_buf(btf, type_id, buf, BPF_DIAG_FMT_BUF_SIZE); + if (ret < 0 || !buf[0]) { + scnprintf(buf, BPF_DIAG_FMT_BUF_SIZE, "BTF type ID %u", type_id); + return buf; + } + + len = strlen(buf); + if (len && buf[len - 1] == '{') + buf[len - 1] = '\0'; + return buf; +} + +static void diag_vprint_indented(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); + +static void diag_vprint_indented(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + char *buf; + + if (!bpf_diag_enabled(env)) + return; + + buf = kvasprintf(GFP_KERNEL_ACCOUNT, fmt, args); + if (!buf) { + diag_write(env, "%s\n", BPF_DIAG_TEXT_INDENT); + return; + } + + diag_print_wrapped_prefixed(env, BPF_DIAG_TEXT_INDENT, BPF_DIAG_TEXT_INDENT, buf); + kfree(buf); +} + +static int diag_line_width(unsigned int line) +{ + int width = 1; + + while (line >= 10) { + line /= 10; + width++; + } + + return width; +} + +static int diag_line_indent(const char *line) +{ + int indent = 0; + + while (*line == ' ' || *line == '\t') { + if (*line == '\t') + indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH); + else + indent++; + line++; + } + + return indent; +} + +static void disasm_print(void *private_data, const char *fmt, ...) __printf(2, 3); + +static void disasm_print(void *private_data, const char *fmt, ...) +{ + struct disasm_ctx *ctx = private_data; + va_list args; + + va_start(args, fmt); + seq_buf_vprintf(&ctx->seq, fmt, args); + va_end(args); +} + +static const char *disasm_kfunc_name(void *private_data, const struct bpf_insn *insn) +{ + struct disasm_ctx *ctx = private_data; + + return bpf_disasm_kfunc_name(ctx->env, insn); +} + +static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, + struct disasm_line *line) +{ + struct disasm_ctx ctx = { .env = env }; + struct bpf_insn *insn; + const struct bpf_insn_cbs cbs = { + .cb_call = disasm_kfunc_name, + .cb_print = disasm_print, + .private_data = &ctx, + }; + + line->idx = insn_idx; + line->valid = false; + seq_buf_init(&ctx.seq, line->text, sizeof(line->text)); + + if (insn_idx < 0 || insn_idx >= env->prog->len) + return; + + if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1])) + return; + + insn = &env->prog->insnsi[insn_idx]; + if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len) + return; + + print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); + seq_buf_str(&ctx.seq); + ctx.seq.len = strnlen(line->text, sizeof(line->text)); + while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') + seq_buf_pop(&ctx.seq); + seq_buf_str(&ctx.seq); + + line->valid = true; +} + +static void diag_format_source_text(char *buf, size_t size, const char *line, int width) +{ + int col = 0, len = 0; + + if (!size) + return; + if (width <= 0) { + buf[0] = '\0'; + return; + } + + line = line ?: "..."; + while (*line && col < width && len + 1 < size) { + if (*line == '\t') { + int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH); + + while (col < next && col < width && len + 1 < size) { + buf[len++] = ' '; + col++; + } + line++; + continue; + } + + buf[len++] = *line++; + col++; + } + + if (*line) { + int ellipsis_len = min(3, width); + + while (len > 0 && col > width - ellipsis_len) { + len--; + col--; + } + while (ellipsis_len-- && len + 1 < size) + buf[len++] = '.'; + } + + buf[len] = '\0'; +} + +static void diag_format_source_lane(char *buf, size_t size, const char *source_prefix, + int source_line_width, int line_num, const char *line) +{ + int len, text_width; + + if (line_num <= 0) { + buf[0] = '\0'; + return; + } + + len = scnprintf(buf, size, "%s%*d | ", source_prefix, source_line_width, line_num); + text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len; + diag_format_source_text(buf + len, size - len, line, text_width); +} + +static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, + const char *problem) +{ + char first; + + if (!bpf_diag_enabled(env)) + return; + + category = category ?: "Verifier Error"; + problem = problem ?: ""; + + if (!problem[0]) { + diag_write(env, "\nVerification failed: %s\n", category); + return; + } + + first = toupper(problem[0]); + diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); +} + +static void diag_reason(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +static void diag_suggestion(struct bpf_verifier_env *env, const char *fmt, ...) + __printf(2, 3); + +static void diag_section(struct bpf_verifier_env *env, const char *title) +{ + if (!bpf_diag_enabled(env)) + return; + + diag_write(env, "\n%s:\n", title); +} + +static void diag_reason(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + diag_section(env, "Reason"); + + va_start(args, fmt); + diag_vprint_indented(env, fmt, args); + va_end(args); +} + +static void diag_suggestion(struct bpf_verifier_env *env, const char *fmt, ...) +{ + va_list args; + + if (!bpf_diag_enabled(env)) + return; + + diag_section(env, "Suggestion"); + + va_start(args, fmt); + diag_vprint_indented(env, fmt, args); + va_end(args); + diag_write(env, "\n"); +} + +static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, + const char *label, const char *msg) +{ + const char *first_prefix, *next_prefix, *text; + + indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8)); + text = bpf_diag_fmt(env, "%s: %s", label, msg); + first_prefix = bpf_diag_fmt(env, " %*s | %*s^-- ", line_width + 4, "", indent, ""); + next_prefix = bpf_diag_fmt(env, " %*s | %*s ", line_width + 4, "", indent, ""); + + diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text); +} + +static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, + struct disasm_line *disasm_lines) +{ + int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0); + int i; + + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + int row = i - BPF_DIAG_CONTEXT; + + format_disasm_line(env, insn_idx + row, &disasm_lines[i]); + } + + diag_write(env, " Instruction context:\n"); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + struct disasm_line *line = &disasm_lines[i]; + + if (line->valid) + diag_write(env, " %s%*d | %s\n", + line->idx == insn_idx ? ">>> " : " ", + insn_width, line->idx, line->text); + } +} + +static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, + const char *fmt, ...) +{ + struct bpf_diag_scratch *scratch; + struct bpf_linfo_source *source_lines; + struct disasm_line *disasm_lines; + struct bpf_linfo_source src = {}; + struct diag_fmt_mark mark; + const struct bpf_line_info *linfo; + const struct bpf_subprog_info *subprog; + struct btf *btf = env->prog->aux->btf; + char *source_lane; + const char *msg; + const char *func; + int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i; + va_list args; + + if (!bpf_diag_enabled(env)) + return; + if (!env->diag) + return; + + mark = diag_fmt_save(env); + label = label ?: "note"; + scratch = &env->diag->scratch; + source_lines = scratch->source_lines; + disasm_lines = scratch->disasm_lines; + memset(source_lines, 0, sizeof(scratch->source_lines)); + memset(disasm_lines, 0, sizeof(scratch->disasm_lines)); + + va_start(args, fmt); + msg = bpf_diag_vfmt(env, fmt, args); + va_end(args); + if (!*msg) + msg = ""; + + linfo = bpf_find_linfo(env->prog, insn_idx); + if (btf && linfo) + bpf_get_linfo_source(btf, linfo, &src); + if (!src.file || !*src.file || !src.line || !*src.line) { + diag_write(env, " insn %u\n", insn_idx); + diag_print_source_annotation(env, 0, 0, label, msg); + diag_print_insn_context(env, insn_idx, disasm_lines); + goto out_restore; + } + + subprog = bpf_find_containing_subprog(env, insn_idx); + subprogno = subprog ? subprog - env->subprog_info : -ENOENT; + func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL; + if (func && *func) + diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); + else + diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); + + start_line = src.line_num - BPF_DIAG_CONTEXT; + end_line = src.line_num + BPF_DIAG_CONTEXT; + width = diag_line_width(end_line); + indent = diag_line_indent(src.line); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) + source_lines[i].line_num = start_line + i; + + linfo = env->prog->aux->linfo; + linfo_start = subprog ? subprog->linfo_idx : 0; + linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ? + env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo; + for (i = linfo_start; i < linfo_end; i++) { + struct bpf_linfo_source line_src; + int idx; + + bpf_get_linfo_source(btf, &linfo[i], &line_src); + if (line_src.file_name_off != src.file_name_off || + line_src.line_num < start_line || line_src.line_num > end_line || + !line_src.line || !*line_src.line) + continue; + + idx = line_src.line_num - start_line; + if (!source_lines[idx].line) + source_lines[idx] = line_src; + } + + diag_write(env, " Source context:\n"); + source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (!source_lane) + goto out_restore; + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + const char *source_prefix; + + source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " "; + diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width, + source_lines[i].line_num, source_lines[i].line); + diag_write(env, " %s\n", source_lane); + if (source_lines[i].line_num == src.line_num) + diag_print_source_annotation(env, width, indent, label, msg); + } + diag_print_insn_context(env, insn_idx, disasm_lines); + +out_restore: + diag_fmt_restore(env, mark); +} + +static const struct bpf_func_state *diag_current_frame(const struct bpf_verifier_env *env) +{ + return env->cur_state->frame[env->cur_state->curframe]; +} + +void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *problem, const char *reason, const char *suggestion) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + if (regno >= 0) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type) +{ + switch (base_type(type)) { + case NOT_INIT: + return "an uninitialized value"; + case SCALAR_VALUE: + return "an integer scalar"; + case PTR_TO_CTX: + return "a context pointer"; + case PTR_TO_STACK: + return "a stack pointer"; + case PTR_TO_MAP_VALUE: + if (type_may_be_null(type)) + return "a nullable map value pointer"; + return "a map value pointer"; + case PTR_TO_MEM: + if (type_may_be_null(type)) + return "a nullable memory pointer"; + return "a memory pointer"; + case PTR_TO_BTF_ID: + if (type_may_be_null(type)) + return "a nullable kernel object pointer"; + if (type_is_non_owning_ref(type)) + return "a borrowed allocated object pointer"; + if (type_is_ptr_alloc_obj(type)) + return "an owned allocated object pointer"; + if (type_flag(type) & PTR_UNTRUSTED) + return "an untrusted kernel object pointer"; + return "a kernel object pointer"; + default: + return reg_type_str(env, type); + } +} + +static const char *diag_arg_ordinal(int argno) +{ + switch (argno) { + case 1: + return "first"; + case 2: + return "second"; + case 3: + return "third"; + case 4: + return "fourth"; + case 5: + return "fifth"; + case 6: + return "sixth"; + case 7: + return "seventh"; + case 8: + return "eighth"; + case 9: + return "ninth"; + case 10: + return "tenth"; + case 11: + return "eleventh"; + case 12: + return "twelfth"; + default: + return NULL; + } +} + +void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, + int stack_arg_slot, const char *call_name, const char *arg_name, + const char *reason, const char *suggestion) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + }; + const char *ordinal = diag_arg_ordinal(argno); + const char *arg_desc; + bool print_history = true; + + if (regno >= 0) { + opts.scope = BPF_DIAG_HISTORY_SCOPE_REG; + opts.regno = regno; + } else if (stack_arg_slot >= 0) { + opts.scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG; + opts.stack_arg_slot = stack_arg_slot; + } else { + print_history = false; + } + + if (ordinal && arg_name) + arg_desc = bpf_diag_fmt(env, "%s argument (%s)", ordinal, arg_name); + else if (ordinal) + arg_desc = bpf_diag_fmt(env, "%s argument", ordinal); + else if (arg_name) + arg_desc = bpf_diag_fmt(env, "argument %s", arg_name); + else + arg_desc = "argument"; + + bpf_diag_header(env, CALL_TYPE_SAFETY, "invalid call argument"); + diag_reason(env, "The %s to %s does not satisfy the verifier contract: %s.", + arg_desc, call_name, reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "invalid %s for %s", arg_desc, call_name); + + if (print_history) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +static const char *diag_context_constraint(enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return "RCU read-side critical sections cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_PREEMPT: + return "preemption-disabled code cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_IRQ: + return "IRQ-disabled code cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_LOCK: + return "code holding a BPF spin lock cannot call operations that may sleep"; + case BPF_DIAG_CONTEXT_NONE: + default: + return NULL; + } +} + +static const char *diag_active_context(struct bpf_verifier_env *env, u32 depth, + const char *context) +{ + if (depth == 1) + return bpf_diag_fmt(env, "an active %s (depth 1)", context); + return bpf_diag_fmt(env, "%u active %ss (depth %u)", depth, context, depth); +} + +static u32 diag_context_depth(struct bpf_verifier_env *env, enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return env->cur_state->active_rcu_locks; + case BPF_DIAG_CONTEXT_PREEMPT: + return env->cur_state->active_preempt_locks; + case BPF_DIAG_CONTEXT_IRQ: + return bpf_diag_irq_depth(env->cur_state); + case BPF_DIAG_CONTEXT_LOCK: + return env->cur_state->active_locks; + case BPF_DIAG_CONTEXT_NONE: + default: + return 0; + } +} + +void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, const char *suggestion) +{ + struct bpf_diag_history_opts opts; + enum bpf_diag_context_kind ctx_kind; + const char *constraint, *context; + u32 depth; + + if (env->cur_state->active_rcu_locks) + ctx_kind = BPF_DIAG_CONTEXT_RCU; + else if (env->cur_state->active_preempt_locks) + ctx_kind = BPF_DIAG_CONTEXT_PREEMPT; + else if (env->cur_state->active_irq_id) + ctx_kind = BPF_DIAG_CONTEXT_IRQ; + else if (env->cur_state->active_locks) + ctx_kind = BPF_DIAG_CONTEXT_LOCK; + else + ctx_kind = BPF_DIAG_CONTEXT_NONE; + + depth = diag_context_depth(env, ctx_kind); + opts = (struct bpf_diag_history_opts) { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + .ctx_depth = depth, + }; + constraint = diag_context_constraint(ctx_kind); + context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, + "operation is not allowed in this context"); + if (constraint) { + if (depth) { + diag_reason( + env, "The operation %s cannot be used in %s because %s. This path is still inside %s.", + operation, context, constraint, diag_active_context(env, depth, context)); + } else { + diag_reason(env, "The operation %s cannot be used in %s because %s.", + operation, context, constraint); + } + } else { + diag_reason(env, "The operation %s cannot be used in %s.", operation, + context); + } + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not allowed in %s", operation, + context); + + if (ctx_kind != BPF_DIAG_CONTEXT_NONE) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_active(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion) +{ + u32 depth = diag_context_depth(env, ctx_kind); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + .ctx_depth = depth, + }; + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, + "operation is not allowed in this context"); + diag_reason( + env, "The operation %s cannot be used while this path is still inside %s. Leave the region before this operation.", + operation, diag_active_context(env, depth, context)); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not allowed before leaving %s", + operation, context); + + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion) +{ + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "required context is not active"); + diag_reason(env, "The operation %s requires an active %s, but this path is outside one.", + operation, context); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s requires %s", operation, context); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, enum bpf_diag_context_kind ctx_kind, + const char *suggestion) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = ctx_kind, + }; + const char *context = diag_context_name(ctx_kind); + + bpf_diag_header(env, EXECUTION_CONTEXT_SAFETY, "unmatched context exit"); + diag_reason( + env, "The operation %s tries to leave %s, but this path has no active %s to leave. The current depth is 0.", + operation, context, context); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s has no matching enter on this path", + operation); + + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, + const char *problem, const char *suggestion, + const char *reason_fmt, ...) +{ + va_list args; + + bpf_diag_header(env, PROGRAM_STRUCTURE, problem); + diag_section(env, "Reason"); + + va_start(args, reason_fmt); + diag_vprint_indented(env, reason_fmt, args); + va_end(args); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_policy(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + const char *reason, const char *suggestion) +{ + bpf_diag_header(env, POLICY, "operation is not allowed"); + diag_reason(env, "The %s is not allowed: %s.", operation, reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "policy check failed for %s", operation); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const struct bpf_reg_state *reg, + enum bpf_diag_invalid_deref_kind kind, s64 offset) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const char *type_name = bpf_diag_reg_type_plain(env, reg->type); + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "invalid dereference"); + + switch (kind) { + case BPF_DIAG_DEREF_SCALAR: + diag_reason(env, "%s is an integer scalar here, not a pointer to memory.", + reg_name); + break; + case BPF_DIAG_DEREF_NULLABLE_PTR: + diag_reason( + env, "%s may be NULL here (%s). The program could dereference NULL on this path, so the verifier cannot prove this access is safe.", + reg_name, type_name); + break; + case BPF_DIAG_DEREF_MODIFIED_PTR: + diag_reason( + env, "%s has offset %lld here, but this pointer type must be dereferenced in its original form.", + reg_name, offset); + break; + case BPF_DIAG_DEREF_INVALID_PTR: + default: + diag_reason( + env, "%s has type %s here, which is not valid for this memory access.", + reg_name, type_name); + break; + } + + diag_section(env, "At"); + if (kind == BPF_DIAG_DEREF_MODIFIED_PTR) + bpf_diag_source(env, insn_idx, "error", + "dereference requires the original %s pointer", type_name); + else + bpf_diag_source(env, insn_idx, "error", "invalid dereference of %s (%s)", + reg_name, type_name); + + if (regno >= 0) + diag_print_history(env, &opts); + + switch (kind) { + case BPF_DIAG_DEREF_NULLABLE_PTR: + diag_suggestion( + env, "Add a NULL check before the access and dereference the pointer only on the non-NULL path."); + break; + case BPF_DIAG_DEREF_MODIFIED_PTR: + diag_suggestion( + env, "Preserve the original pointer in another register, or use only offsets this pointer type permits before dereferencing it."); + break; + case BPF_DIAG_DEREF_SCALAR: + case BPF_DIAG_DEREF_INVALID_PTR: + default: + diag_suggestion( + env, "Preserve a pointer-valued register where needed, or reload and revalidate the pointer after scalar arithmetic, helper calls, or other operations that can invalidate it."); + break; + } +} + +void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const struct bpf_diag_log *log = env->diag ? &env->diag->log : NULL; + struct bpf_diag_mod_target target; + bool invalidated = false; + int i; + + target = diag_reg_target(opts.frame_id, opts.frameno, regno); + for (i = log ? log->cnt : 0; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + + if (event->kind != BPF_DIAG_HISTORY_MOD || + !diag_target_matches(&event->mod.target, &target)) + continue; + invalidated = event->mod.new.type == NOT_INIT; + break; + } + + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "unreadable register"); + if (invalidated) + diag_reason( + env, "R%d is not readable here. A previous operation invalidated this register, so the verifier cannot use it as an input.", + regno); + else if (log && !log->first_seq) + diag_reason(env, + "R%d has never been initialized on this path, so the verifier cannot use it as an input.", + regno); + else + diag_reason( + env, "R%d is not readable here. It may never have been initialized, or an earlier operation may have invalidated it.", + regno); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "R%d is not readable", regno); + + if (regno >= 0) + diag_print_history(env, &opts); + + if (invalidated) + diag_suggestion( + env, "Avoid using the register after it is invalidated, or initialize it again before this instruction."); + else if (log && !log->first_seq) + diag_suggestion(env, "Initialize R%d on every path before this instruction.", regno); + else + diag_suggestion( + env, "Initialize the register on every path, or initialize it again after any operation that invalidates it."); +} + +static int diag_stack_argno(u8 slot) +{ + return MAX_BPF_FUNC_REG_ARGS + slot + 1; +} + +static void diag_format_stack_arg(char *buf, size_t size, u8 slot, const char *arg_name) +{ + int argno = diag_stack_argno(slot); + const char *ordinal = diag_arg_ordinal(argno); + + if (ordinal && arg_name) + scnprintf(buf, size, "outgoing stack argument %u (%s argument, %s)", slot + 1, + ordinal, arg_name); + else if (ordinal) + scnprintf(buf, size, "outgoing stack argument %u (%s argument)", slot + 1, ordinal); + else if (arg_name) + scnprintf(buf, size, "outgoing stack argument %u (%s)", slot + 1, arg_name); + else + scnprintf(buf, size, "outgoing stack argument %u", slot + 1); +} + +void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, + int stack_arg_slot, const char *callee_name, + const char *arg_name) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_STACK_ARG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .stack_arg_slot = stack_arg_slot, + }; + const char *arg_buf; + + arg_buf = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (arg_buf) + diag_format_stack_arg((char *)arg_buf, BPF_DIAG_FMT_BUF_SIZE, stack_arg_slot, + arg_name); + else + arg_buf = ""; + bpf_diag_header(env, REGISTER_TYPE_SAFETY, "missing stack argument"); + if (callee_name && *callee_name) + diag_reason( + env, "Function %s expects %d arguments, but %s is not initialized at this call.", + callee_name, nargs, arg_buf); + else + diag_reason( + env, "The callee expects %d arguments, but %s is not initialized at this call.", + nargs, arg_buf); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s is not initialized", arg_buf); + + if (stack_arg_slot >= 0) + diag_print_history(env, &opts); + + diag_suggestion( + env, "Write the outgoing stack argument after any operation that may invalidate stored pointer values, and before making this call."); +} + +void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion) +{ + bpf_diag_header(env, MEMORY_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_BRANCH, + .branch = { + .cond_true = cond_true, + }, + }; + + diag_append_history(env, &event); +} + +static void diag_snapshot_reg(struct bpf_diag_reg_snapshot *snapshot, + const struct bpf_reg_state *reg) +{ + snapshot->type = reg->type; + if (type_is_map_ptr(reg->type)) + snapshot->map_ptr = reg->map_ptr; + if (base_type(reg->type) == PTR_TO_BTF_ID && reg->btf && reg->btf_id) { + snapshot->btf_id = reg->btf_id; + snapshot->btf = reg->btf; + } + snapshot->var_off = reg->var_off; + snapshot->r64 = reg->r64; +} + +static bool diag_mod_insn_origin(struct bpf_verifier_env *env, u32 insn_idx, + const struct bpf_diag_mod_target *target, + struct bpf_diag_mod_target *origin) +{ + const struct bpf_insn *insn = &env->prog->insnsi[insn_idx]; + u8 class = BPF_CLASS(insn->code); + const struct bpf_func_state *state; + + if (target->kind == BPF_DIAG_MOD_TARGET_REG && (class == BPF_ALU || class == BPF_ALU64) && + BPF_OP(insn->code) == BPF_MOV && BPF_SRC(insn->code) == BPF_X) { + *origin = diag_reg_target(target->frame_id, target->frameno, insn->src_reg); + return true; + } + + if ((target->kind != BPF_DIAG_MOD_TARGET_STACK_ARG && + target->kind != BPF_DIAG_MOD_TARGET_STACK_SLOT) || + class != BPF_STX) + return false; + + state = env->cur_state->frame[env->cur_state->curframe]; + *origin = diag_reg_target(state->diag_frame_id, state->frameno, insn->src_reg); + return true; +} + +static bool diag_mod_keeps_lineage(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + const struct bpf_insn *insn; + u8 class; + + if (event->mod.reason != BPF_DIAG_MOD_WRITE || + event->mod.target.kind != BPF_DIAG_MOD_TARGET_REG) + return false; + + insn = &env->prog->insnsi[event->insn_idx]; + class = BPF_CLASS(insn->code); + if (class != BPF_ALU && class != BPF_ALU64) + return false; + + switch (BPF_OP(insn->code)) { + case BPF_ADD: + case BPF_SUB: + case BPF_MUL: + case BPF_OR: + case BPF_AND: + case BPF_LSH: + case BPF_RSH: + case BPF_ARSH: + case BPF_XOR: + case BPF_NEG: + case BPF_END: + return true; + default: + return false; + } +} + +static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, + struct bpf_diag_mod_target target, + enum bpf_diag_mod_reason reason, + const struct bpf_reg_state *old_reg, + const struct bpf_reg_state *new_reg, + const struct bpf_diag_mod_target *origin) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_MOD, + .mod = { + .target = target, + .reason = reason, + }, + }; + + if (old_reg) + diag_snapshot_reg(&event.mod.old, old_reg); + if (new_reg) + diag_snapshot_reg(&event.mod.new, new_reg); + if (origin) { + event.mod.origin = *origin; + event.mod.origin_valid = true; + } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { + event.mod.origin_valid = true; + } + if (old_reg && new_reg && + (reason == BPF_DIAG_MOD_WRITE || reason == BPF_DIAG_MOD_SPILL) && + !memcmp(&event.mod.old, &event.mod.new, sizeof(event.mod.old)) && + !event.mod.origin_valid && + diag_mod_keeps_lineage(env, &event)) + return; + + diag_append_history(env, &event); +} + +static struct bpf_reg_state *target_to_reg(struct bpf_verifier_env *env, + const struct bpf_diag_mod_target *target) +{ + struct bpf_verifier_state *vstate = env->cur_state; + struct bpf_func_state *state; + + state = target->frameno <= vstate->curframe ? vstate->frame[target->frameno] : NULL; + + if (!state) + return NULL; + if (state->diag_frame_id != target->frame_id) + return NULL; + + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + if (target->regno >= MAX_BPF_REG) + return NULL; + return &state->regs[target->regno]; + case BPF_DIAG_MOD_TARGET_STACK_ARG: + if (target->stack_arg >= state->out_stack_arg_cnt) + return NULL; + return &state->stack_arg_regs[target->stack_arg]; + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + if (target->spi >= state->allocated_stack / BPF_REG_SIZE) + return NULL; + return &state->stack[target->spi].spilled_ptr; + default: + return NULL; + } +} + +static bool reg_to_target(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + struct bpf_diag_mod_target *target) +{ + struct bpf_verifier_state *vstate = env->cur_state; + unsigned long addr = (unsigned long)reg; + int frame; + + for (frame = 0; frame <= vstate->curframe; frame++) { + struct bpf_func_state *state = vstate->frame[frame]; + unsigned long start, end; + u32 nslots = state->allocated_stack / BPF_REG_SIZE; + int spi; + + start = (unsigned long)state->regs; + end = (unsigned long)(state->regs + MAX_BPF_REG); + if (addr >= start && addr < end) { + *target = diag_reg_target(state->diag_frame_id, state->frameno, + reg - state->regs); + return true; + } + + start = (unsigned long)state->stack_arg_regs; + end = (unsigned long)(state->stack_arg_regs + state->out_stack_arg_cnt); + if (state->out_stack_arg_cnt && addr >= start && addr < end) { + *target = diag_stack_arg_target(state->diag_frame_id, state->frameno, + reg - state->stack_arg_regs); + return true; + } + + start = (unsigned long)state->stack; + end = (unsigned long)(state->stack + nslots); + if (nslots && addr >= start && addr < end) { + spi = ((const char *)reg - (const char *)state->stack) / + sizeof(*state->stack); + *target = diag_stack_slot_target(state->diag_frame_id, state->frameno, spi); + return true; + } + } + return false; +} + +void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + diag->mod.active = reg_to_target(env, reg, &diag->mod.target); + if (!diag->mod.active) + return; + diag->mod.target_reg_snapshot = *reg; + diag->mod.insn_idx = env->insn_idx; + diag->mod.reason = reason; + diag->mod.origin_valid = origin && reg_to_target(env, origin, &diag->mod.origin); +} + +void bpf_diag_mod_end(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + const struct bpf_reg_state *new_reg; + + if (!diag || !diag->mod.active) + return; + diag->mod.active = false; + /* + * Resolve the target again because the enclosing function state's stack + * may have been reallocated while the modification was in progress. + */ + new_reg = target_to_reg(env, &diag->mod.target); + if (!new_reg) + return; + diag_record_mod(env, diag->mod.insn_idx, diag->mod.target, diag->mod.reason, + &diag->mod.target_reg_snapshot, new_reg, + diag->mod.origin_valid ? &diag->mod.origin : NULL); +} + +void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + enum bpf_diag_mod_reason reason) +{ + struct bpf_diag_mod_target target; + + if (!env->diag || reg->type == NOT_INIT || !reg_to_target(env, reg, &target)) + return; + diag_record_mod(env, env->insn_idx, target, reason, reg, NULL, NULL); +} + +void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, + const struct bpf_func_state *state, s16 min_off, s16 max_off, + enum bpf_diag_mod_reason reason) +{ + diag_record_mod(env, env->insn_idx, + diag_stack_range_target(state->diag_frame_id, state->frameno, min_off, max_off), + reason, NULL, NULL, NULL); +} + +static void diag_record_ref(struct bpf_verifier_env *env, u32 insn_idx, u8 kind, u32 ref_id) +{ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = kind, + .ref = { + .ref_id = ref_id, + }, + }; + + diag_append_history(env, &event); +} + +void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) +{ + diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_ACQUIRE, ref_id); +} + +void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id) +{ + diag_record_ref(env, insn_idx, BPF_DIAG_HISTORY_REF_RELEASE, ref_id); +} + +void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, + enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth) +{ + /* + * Keep leave events so context rendering can stop at a depth-zero exit + * and show nested-region depth accurately for the active path. + */ + struct bpf_diag_history_event event = { + .insn_idx = insn_idx, + .kind = BPF_DIAG_HISTORY_CONTEXT, + .ctx = { + .kind = ctx_kind, + .enter = enter, + .depth = depth, + }, + }; + + diag_append_history(env, &event); +} + +static int diag_history_context_start_idx(const struct bpf_diag_log *log, + const struct bpf_diag_history_opts *opts) +{ + int i; + + if (!opts->ctx_depth) + return 0; + + /* Find the most recent outermost entry, or a depth-zero exit. */ + for (i = log->cnt; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + + if (event->kind != BPF_DIAG_HISTORY_CONTEXT || event->ctx.kind != opts->ctx_kind) + continue; + + if (event->ctx.enter && event->ctx.depth == 1) + return i - 1; + if (!event->ctx.enter && event->ctx.depth == 0) + return 0; + } + + return 0; +} + +struct bpf_diag_history_filter { + const struct bpf_diag_history_opts *opts; + u32 lineage_start; + bool lineage_valid; +}; + +static bool diag_target_matches(const struct bpf_diag_mod_target *event_target, + const struct bpf_diag_mod_target *target) +{ + int slot_off; + + if (event_target->frame_id != target->frame_id || event_target->frameno != target->frameno) + return false; + + if (event_target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE && + target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) { + slot_off = -(target->spi + 1) * BPF_REG_SIZE; + return event_target->range.min_off < slot_off + BPF_REG_SIZE && + event_target->range.max_off > slot_off; + } + + if (event_target->kind != target->kind) + return false; + + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + return event_target->regno == target->regno; + case BPF_DIAG_MOD_TARGET_STACK_ARG: + return event_target->stack_arg == target->stack_arg; + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + return event_target->spi == target->spi; + default: + return false; + } +} + +static void diag_build_lineage(struct bpf_verifier_env *env, struct bpf_diag_log *log, + struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + struct bpf_diag_mod_target target; + int i; + + for (i = 0; i < log->cnt; i++) + log->events[log_pos(log, i)].in_lineage = false; + + if (opts->scope == BPF_DIAG_HISTORY_SCOPE_REG) + target = diag_reg_target(opts->frame_id, opts->frameno, opts->regno); + else if (opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG) + target = diag_stack_arg_target(opts->frame_id, opts->frameno, + opts->stack_arg_slot); + else + return; + + /* + * Find the nearest mutation of the active target. A fill or spill changes + * the target to its origin, so the same walk follows register/stack + * lineage recursively until it reaches the write that created the value. + */ + for (i = log->cnt; i > 0; i--) { + struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + if (event->kind != BPF_DIAG_HISTORY_MOD || + !diag_target_matches(&event->mod.target, &target)) + continue; + + event->in_lineage = true; + filter->lineage_start = i - 1; + filter->lineage_valid = true; + + if (event->mod.origin_valid) { + target = event->mod.origin; + continue; + } + if (event->mod.reason != BPF_DIAG_MOD_WRITE && + event->mod.reason != BPF_DIAG_MOD_SPILL) + continue; + if (diag_mod_keeps_lineage(env, event)) + continue; + break; + } +} + +static int diag_history_start_idx(const struct bpf_diag_log *log, + const struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + int i; + + if (opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT) + return diag_history_context_start_idx(log, opts); + if (filter->lineage_valid) + return filter->lineage_start; + if (opts->scope != BPF_DIAG_HISTORY_SCOPE_REF) + return 0; + + for (i = log->cnt; i > 0; i--) { + const struct bpf_diag_history_event *event; + + event = &log->events[log_pos(log, i - 1)]; + if (event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE && + event->ref.ref_id == opts->ref_id) + return i - 1; + } + + return 0; +} + +static bool diag_history_event_visible(const struct bpf_diag_history_event *event, + const struct bpf_diag_history_filter *filter) +{ + const struct bpf_diag_history_opts *opts = filter->opts; + + switch (event->kind) { + case BPF_DIAG_HISTORY_BRANCH: + return true; + case BPF_DIAG_HISTORY_MOD: + return filter->lineage_valid && event->in_lineage; + case BPF_DIAG_HISTORY_REF_ACQUIRE: + case BPF_DIAG_HISTORY_REF_RELEASE: + return opts->scope == BPF_DIAG_HISTORY_SCOPE_REF && + event->ref.ref_id == opts->ref_id; + case BPF_DIAG_HISTORY_CONTEXT: + return opts->scope == BPF_DIAG_HISTORY_SCOPE_CONTEXT && + event->ctx.kind == opts->ctx_kind; + default: + return false; + } +} + +static const char *diag_s64_bound_name(s64 value) +{ + if (value == S64_MIN) + return "S64_MIN"; + if (value == S64_MAX) + return "S64_MAX"; + return NULL; +} + +static const char *diag_u64_bound_name(u64 value) +{ + if (value == U64_MAX) + return "U64_MAX"; + return NULL; +} + +static const char *diag_s64_str(struct bpf_verifier_env *env, s64 value) +{ + return diag_s64_bound_name(value) ?: bpf_diag_fmt(env, "%lld", value); +} + +static const char *diag_u64_str(struct bpf_verifier_env *env, u64 value) +{ + return diag_u64_bound_name(value) ?: bpf_diag_fmt(env, "%llu", value); +} + +static bool diag_cnum64_unknown(struct cnum64 range) +{ + return cnum64_smin(range) == S64_MIN && cnum64_smax(range) == S64_MAX && + cnum64_umin(range) == 0 && cnum64_umax(range) == U64_MAX; +} + +static bool diag_snapshot_unknown(const struct bpf_diag_reg_snapshot *snapshot) +{ + return tnum_is_unknown(snapshot->var_off) && diag_cnum64_unknown(snapshot->r64); +} + +static const char *diag_scalar_range(struct bpf_verifier_env *env, struct cnum64 range) +{ + return bpf_diag_fmt(env, "signed range [%s, %s], unsigned range [%s, %s]", + diag_s64_str(env, cnum64_smin(range)), + diag_s64_str(env, cnum64_smax(range)), + diag_u64_str(env, cnum64_umin(range)), + diag_u64_str(env, cnum64_umax(range))); +} + +const char *bpf_diag_fmt_s64_sum(struct bpf_verifier_env *env, s64 value, int addend) +{ + s64 sum; + + if (check_add_overflow(value, (s64)addend, &sum)) + return bpf_diag_fmt(env, "%lld plus %d (%s)", value, addend, + addend < 0 ? "below S64_MIN" : "above S64_MAX"); + + return bpf_diag_fmt(env, "%lld", sum); +} + +static const char *diag_access_offset(struct bpf_verifier_env *env, int off, + const struct bpf_reg_state *reg) +{ + if (tnum_is_const(reg->var_off)) + return bpf_diag_fmt(env, "constant %s", + bpf_diag_fmt_s64_sum(env, (s64)reg->var_off.value, off)); + + if (tnum_is_unknown(reg->var_off) && diag_cnum64_unknown(reg->r64)) + return bpf_diag_fmt(env, "unbounded"); + + if (off) + return bpf_diag_fmt(env, + "variable: known bits %#llx, unknown mask %#llx, plus fixed offset %d; %s", + (u64)reg->var_off.value, reg->var_off.mask, off, + diag_scalar_range(env, reg->r64)); + return bpf_diag_fmt(env, "variable: known bits %#llx, unknown mask %#llx; %s", + (u64)reg->var_off.value, reg->var_off.mask, + diag_scalar_range(env, reg->r64)); +} + +void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const char *type_name, const char *proof, + int off, int size, u32 mem_size, const struct bpf_reg_state *reg) +{ + const struct bpf_func_state *frame = diag_current_frame(env); + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REG, + .frame_id = frame->diag_frame_id, + .frameno = frame->frameno, + .regno = regno, + }; + const char *offset_desc; + + if (!bpf_diag_enabled(env)) + return; + + offset_desc = diag_access_offset(env, off, reg); + + bpf_diag_header(env, MEMORY_SAFETY, "access outside bounds"); + diag_reason( + env, "The verifier cannot prove offset + access_size <= object_size. Here, %s. %s is %s; offset is %s; access_size is %d; object_size is %u.", + proof, reg_name, type_name, offset_desc, size, mem_size); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "access may be outside object bounds"); + + if (regno >= 0) + diag_print_history(env, &opts); + + diag_suggestion( + env, "Add or adjust a bounds check that proves offset + access_size stays within the object."); +} + +static const char *diag_lock_name(const struct bpf_reference_state *lock) +{ + switch (lock->type) { + case REF_TYPE_LOCK: + return "bpf_spin_lock"; + case REF_TYPE_RES_LOCK: + return "resource spin lock"; + case REF_TYPE_RES_LOCK_IRQ: + return "IRQ-saving resource spin lock"; + default: + return "lock"; + } +} + +static void diag_res_report(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason) +{ + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); +} + +void bpf_diag_res(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion) +{ + diag_res_report(env, insn_idx, problem, reason); + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, + const struct bpf_reference_state *active_lock) +{ + diag_res_report(env, insn_idx, problem, reason); + + if (active_lock) { + diag_section(env, "Active lock"); + bpf_diag_source(env, active_lock->insn_idx, "acquired", + "active %s has verifier identity %d", + diag_lock_name(active_lock), active_lock->id); + } + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, u32 depth) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_CONTEXT, + .ctx_kind = BPF_DIAG_CONTEXT_IRQ, + .ctx_depth = depth, + }; + + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, problem); + diag_reason(env, "%s", reason); + + diag_section(env, "At"); + bpf_diag_source(env, insn_idx, "error", "%s", problem); + + if (depth) + diag_print_history(env, &opts); + + diag_suggestion(env, "%s", suggestion); +} + +void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn) +{ + struct bpf_diag_history_opts opts = { + .scope = BPF_DIAG_HISTORY_SCOPE_REF, + .ref_id = ref_id, + }; + + bpf_diag_header(env, RESOURCE_LIFETIME_SAFETY, "unreleased resource"); + diag_reason( + env, "Owned resource (id=%u) was acquired at instruction %u and still needs to be released before this exit path.", + ref_id, alloc_insn); + + diag_section(env, "At"); + bpf_diag_source(env, fail_insn, "error", + "owned resource (id=%u) still needs release", ref_id); + + diag_print_history(env, &opts); + + diag_suggestion( + env, "Release or transfer ownership of the acquired resource on every path before the program exits."); +} + +static const char *diag_var_offset(struct bpf_verifier_env *env, + const struct bpf_diag_reg_snapshot *snapshot) +{ + if (tnum_is_const(snapshot->var_off)) + return bpf_diag_fmt(env, "at offset %lld", (s64)snapshot->var_off.value); + + if (diag_snapshot_unknown(snapshot)) + return bpf_diag_fmt(env, "with unknown offset"); + + return bpf_diag_fmt(env, + "with variable offset: known bits %#llx, unknown mask %#llx, %s", + snapshot->var_off.value, snapshot->var_off.mask, + diag_scalar_range(env, snapshot->r64)); +} + +static const char *diag_reg_map_name(const struct bpf_map *map) +{ + if (!map || !map->name[0]) + return NULL; + + return map->name; +} + +static const char *diag_reg_snapshot(struct bpf_verifier_env *env, + const struct bpf_diag_reg_snapshot *snapshot) +{ + const char *type_name = reg_type_str(env, snapshot->type); + const char *offset = diag_var_offset(env, snapshot); + const char *btf = snapshot->btf && snapshot->btf_id ? + bpf_diag_fmt_btf_type(env, snapshot->btf, snapshot->btf_id) : NULL; + const char *map_name; + + if (snapshot->type == SCALAR_VALUE) { + if (tnum_is_const(snapshot->var_off)) + return bpf_diag_fmt(env, "integer scalar value %lld", + (s64)snapshot->var_off.value); + if (diag_snapshot_unknown(snapshot)) + return bpf_diag_fmt(env, "integer scalar with unknown value"); + if (cnum64_is_const(snapshot->r64)) + return bpf_diag_fmt(env, "integer scalar value %lld", + cnum64_smin(snapshot->r64)); + return bpf_diag_fmt(env, "integer scalar with %s", + diag_scalar_range(env, snapshot->r64)); + } + + if (snapshot->type == NOT_INIT) + return bpf_diag_fmt(env, "uninitialized value"); + + if (base_type(snapshot->type) == PTR_TO_CTX) + return bpf_diag_fmt(env, "context pointer %s", offset); + + if (base_type(snapshot->type) == PTR_TO_STACK) + return bpf_diag_fmt(env, "stack pointer %s", offset); + + if (base_type(snapshot->type) == PTR_TO_MAP_VALUE) { + const char *kind = type_may_be_null(snapshot->type) ? "nullable map value" : + "map value"; + + map_name = diag_reg_map_name(snapshot->map_ptr); + if (map_name) + return bpf_diag_fmt(env, "%s from %s %s", kind, map_name, offset); + return bpf_diag_fmt(env, "%s %s", kind, offset); + } + + if (base_type(snapshot->type) == CONST_PTR_TO_MAP) { + map_name = diag_reg_map_name(snapshot->map_ptr); + if (map_name) + return bpf_diag_fmt(env, "map pointer for map %s", map_name); + return bpf_diag_fmt(env, "map pointer"); + } + + if (type_is_non_owning_ref(snapshot->type)) { + if (btf) + return bpf_diag_fmt(env, "borrowed allocated object pointer type=%s", btf); + return bpf_diag_fmt(env, "borrowed allocated object pointer"); + } + + if (type_is_ptr_alloc_obj(snapshot->type)) { + if (btf) + return bpf_diag_fmt(env, "owned allocated object pointer type=%s", btf); + return bpf_diag_fmt(env, "owned allocated object pointer"); + } + + if (base_type(snapshot->type) == PTR_TO_BTF_ID && btf) + return bpf_diag_fmt(env, "%s type=%s %s", type_name, btf, offset); + + return bpf_diag_fmt(env, "%s %s", type_name, offset); +} + +static const char *diag_mod_target_desc(struct bpf_verifier_env *env, + const struct bpf_diag_mod_target *target) +{ + switch (target->kind) { + case BPF_DIAG_MOD_TARGET_REG: + return bpf_diag_fmt(env, "R%u", target->regno); + case BPF_DIAG_MOD_TARGET_STACK_ARG: + return bpf_diag_fmt(env, "stack arg%d", diag_stack_argno(target->stack_arg)); + case BPF_DIAG_MOD_TARGET_STACK_SLOT: + return bpf_diag_fmt(env, "stack slot fp%d", -(target->spi + 1) * BPF_REG_SIZE); + default: + return "value"; + } +} + +static void diag_print_mod(struct bpf_verifier_env *env, const struct bpf_diag_history_event *event) +{ + const struct bpf_diag_mod_target *target = &event->mod.target; + const char *target_desc, *reason = NULL, *old, *new; + const char *label = "update"; + + if (target->kind == BPF_DIAG_MOD_TARGET_STACK_RANGE) { + bpf_diag_source( + env, event->insn_idx, "invalidated", + "variable-offset stack write may affect bytes fp%d through fp%d", + target->range.min_off, target->range.max_off - 1); + return; + } + + old = diag_reg_snapshot(env, &event->mod.old); + new = diag_reg_snapshot(env, &event->mod.new); + target_desc = diag_mod_target_desc(env, target); + + switch (event->mod.reason) { + case BPF_DIAG_MOD_REF_RELEASE: + reason = target->kind == BPF_DIAG_MOD_TARGET_REG ? "resource release invalidated " + "this pointer" : + "resource release invalidated " + "this value"; + break; + case BPF_DIAG_MOD_PKT_DATA_CHANGE: + reason = "packet data may have moved"; + break; + case BPF_DIAG_MOD_NON_OWN_REF: + reason = "leaving the protected region invalidated this borrowed pointer"; + break; + case BPF_DIAG_MOD_CALLER_SAVED: + reason = target->kind == BPF_DIAG_MOD_TARGET_STACK_ARG ? + "call invalidated this outgoing stack argument" : + "call invalidated this caller-saved register"; + break; + case BPF_DIAG_MOD_WRITE: + if (target->kind == BPF_DIAG_MOD_TARGET_STACK_SLOT) + reason = "a later stack write overwrote this spilled value"; + break; + case BPF_DIAG_MOD_SPILL: + label = "spilled"; + break; + case BPF_DIAG_MOD_VAR_WRITE: + default: + break; + } + + if (reason) { + bpf_diag_source(env, event->insn_idx, "invalidated", + "%s: %s; previous value was %s", target_desc, reason, old); + return; + } + + bpf_diag_source(env, event->insn_idx, label, "%s changed from %s to %s", target_desc, + old, new); +} + +static void diag_print_ref_event(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + const char *label; + + label = event->kind == BPF_DIAG_HISTORY_REF_ACQUIRE ? "acquired" : "released"; + bpf_diag_source(env, event->insn_idx, label, "owned resource (id=%u)", + event->ref.ref_id); +} + +static const char *diag_context_name(enum bpf_diag_context_kind kind) +{ + switch (kind) { + case BPF_DIAG_CONTEXT_RCU: + return "RCU read lock region"; + case BPF_DIAG_CONTEXT_PREEMPT: + return "non-preemptible region"; + case BPF_DIAG_CONTEXT_IRQ: + return "IRQ-disabled region"; + case BPF_DIAG_CONTEXT_LOCK: + return "lock region"; + case BPF_DIAG_CONTEXT_NONE: + default: + return "non-sleepable program"; + } +} + +static void diag_print_context_event(struct bpf_verifier_env *env, + const struct bpf_diag_history_event *event) +{ + bpf_diag_source(env, event->insn_idx, "context", "%s %s; depth is now %u", + event->ctx.enter ? "entered" : "left", + diag_context_name(event->ctx.kind), event->ctx.depth); +} + +static void diag_print_history(struct bpf_verifier_env *env, + const struct bpf_diag_history_opts *opts) +{ + const struct bpf_diag_history_event *event; + struct bpf_diag_history_filter filter = { + .opts = opts, + }; + struct bpf_diag_log *log; + struct diag_fmt_mark mark; + bool first = true; + int start_idx; + u32 i, visible_cnt = 0, visible_idx = 0; + + if (!bpf_diag_enabled(env)) + return; + + if (!env->diag) + return; + log = &env->diag->log; + + diag_build_lineage(env, log, &filter); + + start_idx = diag_history_start_idx(log, &filter); + for (i = start_idx; i < log->cnt; i++) { + event = &log->events[log_pos(log, i)]; + if (diag_history_event_visible(event, &filter)) + visible_cnt++; + } + + if (!visible_cnt && !log->first_seq && opts->scope == BPF_DIAG_HISTORY_SCOPE_STACK_ARG) + return; + + diag_section(env, "Causal path"); + mark = diag_fmt_save(env); + for (i = start_idx; i < log->cnt; i++) { + event = &log->events[log_pos(log, i)]; + if (!diag_history_event_visible(event, &filter)) + continue; + + diag_fmt_restore(env, mark); + if (visible_cnt > BPF_DIAG_HISTORY_RENDER_MAX && + visible_idx >= BPF_DIAG_HISTORY_RENDER_MAX / 2 && + visible_idx < visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX / 2) { + if (visible_idx++ != BPF_DIAG_HISTORY_RENDER_MAX / 2) + continue; + if (!first) + diag_write(env, "\n"); + first = false; + diag_write(env, " %u intermediate causal-history events omitted\n", + visible_cnt - BPF_DIAG_HISTORY_RENDER_MAX); + continue; + } + visible_idx++; + + if (!first) + diag_write(env, "\n"); + first = false; + + switch (event->kind) { + case BPF_DIAG_HISTORY_BRANCH: + bpf_diag_source(env, event->insn_idx, "branch", + "took the %s branch of this conditional, goto %s", + event->branch.cond_true ? "true" : "false", + event->branch.cond_true ? "followed" : "not followed"); + break; + case BPF_DIAG_HISTORY_MOD: + diag_print_mod(env, event); + break; + case BPF_DIAG_HISTORY_REF_ACQUIRE: + case BPF_DIAG_HISTORY_REF_RELEASE: + diag_print_ref_event(env, event); + break; + case BPF_DIAG_HISTORY_CONTEXT: + diag_print_context_event(env, event); + break; + default: + break; + } + } + + if (!visible_cnt) + diag_write(env, " no retained diagnostic events on this path\n"); + if (log->first_seq) + diag_write(env, " %llu older causal-history event%s not retained because diagnostic " + "event storage reached capacity\n", + log->first_seq, log->first_seq == 1 ? "" : "s"); + diag_fmt_restore(env, mark); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h new file mode 100644 index 000000000000..d1b79945008a --- /dev/null +++ b/kernel/bpf/diagnostics.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef __BPF_DIAGNOSTICS_H +#define __BPF_DIAGNOSTICS_H + +#include +#include +#include +#include + +struct bpf_reference_state; +struct bpf_func_state; +struct bpf_reg_state; +struct bpf_verifier_env; +struct bpf_verifier_state; +struct btf; + +const char *bpf_diag_fmt_s64_sum(struct bpf_verifier_env *env, s64 value, int addend); +enum bpf_diag_mod_reason { + BPF_DIAG_MOD_WRITE, + BPF_DIAG_MOD_SPILL, + BPF_DIAG_MOD_VAR_WRITE, + BPF_DIAG_MOD_REF_RELEASE, + BPF_DIAG_MOD_PKT_DATA_CHANGE, + BPF_DIAG_MOD_NON_OWN_REF, + BPF_DIAG_MOD_CALLER_SAVED, +}; + +enum bpf_diag_context_kind { + BPF_DIAG_CONTEXT_NONE, + BPF_DIAG_CONTEXT_RCU, + BPF_DIAG_CONTEXT_PREEMPT, + BPF_DIAG_CONTEXT_IRQ, + BPF_DIAG_CONTEXT_LOCK, +}; + +enum bpf_diag_invalid_deref_kind { + BPF_DIAG_DEREF_SCALAR, + BPF_DIAG_DEREF_NULLABLE_PTR, + BPF_DIAG_DEREF_MODIFIED_PTR, + BPF_DIAG_DEREF_INVALID_PTR, +}; + +bool bpf_diag_enabled(const struct bpf_verifier_env *env); +int bpf_diag_init(struct bpf_verifier_env *env); +void bpf_diag_init_frame(struct bpf_verifier_env *env, struct bpf_func_state *state); +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size); +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +const char *bpf_diag_fmt_btf_type(struct bpf_verifier_env *env, const struct btf *btf, u32 type_id); +const char *bpf_diag_reg_type_plain(struct bpf_verifier_env *env, enum bpf_reg_type type); +u64 bpf_diag_event_log_save(struct bpf_verifier_env *env); +void bpf_diag_event_log_restore(struct bpf_verifier_env *env, u64 log_pos); +u32 bpf_diag_irq_depth(const struct bpf_verifier_state *state); +void bpf_diag_free(struct bpf_verifier_env *env); +void bpf_diag_register_type(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *problem, const char *reason, const char *suggestion); +void bpf_diag_invalid_deref(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const struct bpf_reg_state *reg, + enum bpf_diag_invalid_deref_kind kind, s64 offset); +void bpf_diag_unreadable_reg(struct bpf_verifier_env *env, u32 insn_idx, int regno); +void bpf_diag_stack_arg_uninit(struct bpf_verifier_env *env, u32 insn_idx, int nargs, + int stack_arg_slot, const char *callee_name, + const char *arg_name); +void bpf_diag_memory(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion); +void bpf_diag_mem_bounds(struct bpf_verifier_env *env, u32 insn_idx, int regno, + const char *reg_name, const char *type_name, const char *proof, + int off, int size, u32 mem_size, const struct bpf_reg_state *reg); +void bpf_diag_res(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion); +void bpf_diag_lock(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, + const struct bpf_reference_state *active_lock); +void bpf_diag_irq(struct bpf_verifier_env *env, u32 insn_idx, const char *problem, + const char *reason, const char *suggestion, u32 depth); +void bpf_diag_leak(struct bpf_verifier_env *env, u32 ref_id, u32 alloc_insn, u32 fail_insn); +void bpf_diag_call_type(struct bpf_verifier_env *env, u32 insn_idx, int argno, int regno, + int stack_arg_slot, const char *call_name, const char *arg_name, + const char *reason, const char *suggestion); +void bpf_diag_ctx_forbidden(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, const char *suggestion); +void bpf_diag_ctx_active(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_ctx_required(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + enum bpf_diag_context_kind ctx_kind, const char *suggestion); +void bpf_diag_ctx_underflow(struct bpf_verifier_env *env, u32 insn_idx, + const char *operation, enum bpf_diag_context_kind ctx_kind, + const char *suggestion); +void bpf_diag_program_structure(struct bpf_verifier_env *env, u32 insn_idx, + const char *problem, const char *suggestion, + const char *reason_fmt, ...) __printf(5, 6); +void bpf_diag_policy(struct bpf_verifier_env *env, u32 insn_idx, const char *operation, + const char *reason, const char *suggestion); +void bpf_diag_record_branch(struct bpf_verifier_env *env, u32 insn_idx, bool cond_true); +void bpf_diag_mod_begin(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + const struct bpf_reg_state *origin, enum bpf_diag_mod_reason reason); +void bpf_diag_mod_end(struct bpf_verifier_env *env); +void bpf_diag_record_scrub(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, + enum bpf_diag_mod_reason reason); +void bpf_diag_record_scrub_stack(struct bpf_verifier_env *env, + const struct bpf_func_state *state, s16 min_off, s16 max_off, + enum bpf_diag_mod_reason reason); +void bpf_diag_record_ref_acquire(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); +void bpf_diag_record_ref_release(struct bpf_verifier_env *env, u32 insn_idx, u32 ref_id); +void bpf_diag_record_context(struct bpf_verifier_env *env, u32 insn_idx, + enum bpf_diag_context_kind ctx_kind, bool enter, u32 depth); + +#endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index b740fa73ee26..589770ca3d3a 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -615,17 +615,6 @@ static void print_scalar_ranges(struct bpf_verifier_env *env, } } -static bool type_is_map_ptr(enum bpf_reg_type t) { - switch (base_type(t)) { - case CONST_PTR_TO_MAP: - case PTR_TO_MAP_KEY: - case PTR_TO_MAP_VALUE: - return true; - default: - return false; - } -} - /* * _a stands for append, was shortened to avoid multiline statements below. * This macro is used to output a comma separated list of attributes. diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6ac1afced20b..0bb7ee95c8bd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -34,6 +34,7 @@ #include #include +#include "diagnostics.h" #include "disasm.h" static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { @@ -193,6 +194,7 @@ struct bpf_verifier_stack_elem { struct bpf_verifier_stack_elem *next; /* length of verifier log at the time this state was pushed on stack */ u32 log_pos; + u64 diag_log_pos; }; #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192 @@ -203,7 +205,8 @@ struct bpf_verifier_stack_elem { #define BPF_PRIV_STACK_MIN_SIZE 64 static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int parent_id); -static int release_reference_nomark(struct bpf_verifier_state *state, int id); +static int __release_reference_nomark(struct bpf_verifier_state *state, int id); +static int release_reference_nomark(struct bpf_verifier_env *env, int id); static int release_reference(struct bpf_verifier_env *env, int id); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); @@ -405,7 +408,7 @@ static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) return btf_type_is_void(type); } -static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) { struct bpf_func_info *info; @@ -813,6 +816,10 @@ static int destroy_if_dynptr_stack_slot(struct bpf_verifier_env *env, if (dynptr_type_referenced(state->stack[spi].spilled_ptr.dynptr.type) && dynptr_ref_cnt(env, state->stack[spi].spilled_ptr.parent_id) <= 1) { verbose(env, "cannot overwrite referenced dynptr\n"); + bpf_diag_res( + env, env->insn_idx, "referenced dynptr overwrite", + "This stack slot contains a dynptr that owns or protects a referenced resource. Overwriting the last dynptr for that resource would lose the verifier-tracked release path.", + "Release or clone the dynptr so another live dynptr still tracks the referenced resource before overwriting this stack slot."); return -EINVAL; } @@ -883,26 +890,29 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re return true; } +static enum bpf_dynptr_type dynptr_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg) +{ + struct bpf_func_state *state; + int spi; + + if (reg->type == CONST_PTR_TO_DYNPTR) + return reg->dynptr.type; + + spi = dynptr_get_spi(env, reg); + if (spi < 0) + return BPF_DYNPTR_TYPE_INVALID; + state = bpf_func(env, reg); + return state->stack[spi].spilled_ptr.dynptr.type; +} + static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type) { - struct bpf_func_state *state = bpf_func(env, reg); - enum bpf_dynptr_type dynptr_type; - int spi; - /* ARG_PTR_TO_DYNPTR takes any type of dynptr */ if (arg_type == ARG_PTR_TO_DYNPTR) return true; - dynptr_type = arg_to_dynptr_type(arg_type); - if (reg->type == CONST_PTR_TO_DYNPTR) { - return reg->dynptr.type == dynptr_type; - } else { - spi = dynptr_get_spi(env, reg); - if (spi < 0) - return false; - return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type; - } + return dynptr_reg_type(env, reg) == arg_to_dynptr_type(arg_type); } static void __mark_reg_known_zero(struct bpf_reg_state *reg); @@ -1042,7 +1052,7 @@ static int is_iter_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_s } static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx); -static int release_irq_state(struct bpf_verifier_state *state, int id); +static int release_irq_state(struct bpf_verifier_env *env, int id); static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, @@ -1095,13 +1105,23 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r if (st->irq.kfunc_class != kfunc_class) { const char *flag_kfunc = st->irq.kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; const char *used_kfunc = kfunc_class == IRQ_NATIVE_KFUNC ? "native" : "lock"; + const char *reason; verbose(env, "irq flag acquired by %s kfuncs cannot be restored with %s kfuncs\n", flag_kfunc, used_kfunc); + reason = bpf_diag_fmt(env, + "This IRQ flag was saved by %s IRQ kfuncs, but the restore call " + "belongs to the %s IRQ kfunc family. Save and restore operations " + "must use the same family.", + flag_kfunc, used_kfunc); + bpf_diag_irq(env, env->insn_idx, "IRQ flag restore mismatch", reason, + "Restore the flag with the matching IRQ restore kfunc for the save " + "operation that created it.", + bpf_diag_irq_depth(env->cur_state)); return -EINVAL; } - err = release_irq_state(env->cur_state, st->id); + err = release_irq_state(env, st->id); WARN_ON_ONCE(err && err != -EACCES); if (err) { int insn_idx = 0; @@ -1115,6 +1135,11 @@ static int unmark_stack_slot_irq_flag(struct bpf_verifier_env *env, struct bpf_r verbose(env, "cannot restore irq state out of order, expected id=%d acquired at insn_idx=%d\n", env->cur_state->active_irq_id, insn_idx); + bpf_diag_irq(env, env->insn_idx, "IRQ flag restore out of order", + "IRQ-disabled regions must be restored in last-in, first-out order, " + "but this restore does not match the currently active IRQ flag.", + "Restore nested IRQ flags in the reverse order they were saved.", + bpf_diag_irq_depth(env->cur_state)); return err; } @@ -1416,6 +1441,7 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par s->type = REF_TYPE_PTR; s->id = ++env->id_gen; s->parent_id = parent_id; + bpf_diag_record_ref_acquire(env, insn_idx, s->id); return s->id; } @@ -1435,6 +1461,8 @@ static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum r state->active_locks++; state->active_lock_id = id; state->active_lock_ptr = ptr; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_LOCK, true, + state->active_locks); return 0; } @@ -1450,6 +1478,8 @@ static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx) s->id = ++env->id_gen; state->active_irq_id = s->id; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_IRQ, true, + bpf_diag_irq_depth(state)); return s->id; } @@ -1491,8 +1521,9 @@ static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg return find_reference_state(env->cur_state, reg->id); } -static int release_lock_state(struct bpf_verifier_state *state, int type, int id, void *ptr) +static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) { + struct bpf_verifier_state *state = env->cur_state; void *prev_ptr = NULL; u32 prev_id = 0; int i; @@ -1505,6 +1536,8 @@ static int release_lock_state(struct bpf_verifier_state *state, int type, int id /* Reassign active lock (id, ptr). */ state->active_lock_id = prev_id; state->active_lock_ptr = prev_ptr; + bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_LOCK, + false, state->active_locks); return 0; } if (state->refs[i].type & REF_TYPE_LOCK_MASK) { @@ -1515,8 +1548,9 @@ static int release_lock_state(struct bpf_verifier_state *state, int type, int id return -EINVAL; } -static int release_irq_state(struct bpf_verifier_state *state, int id) +static int release_irq_state(struct bpf_verifier_env *env, int id) { + struct bpf_verifier_state *state = env->cur_state; u32 prev_id = 0; int i; @@ -1529,6 +1563,8 @@ static int release_irq_state(struct bpf_verifier_state *state, int id) if (state->refs[i].id == id) { release_reference_state(state, i); state->active_irq_id = prev_id; + bpf_diag_record_context(env, env->insn_idx, BPF_DIAG_CONTEXT_IRQ, + false, bpf_diag_irq_depth(state)); return 0; } else { prev_id = state->refs[i].id; @@ -1699,6 +1735,7 @@ static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, err = bpf_copy_verifier_state(cur, &head->st); if (err) return err; + bpf_diag_event_log_restore(env, head->diag_log_pos); } if (pop_log) bpf_vlog_reset(&env->log, head->log_pos); @@ -1742,6 +1779,7 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; + elem->diag_log_pos = bpf_diag_event_log_save(env); env->head = elem; env->stack_size++; err = bpf_copy_verifier_state(&elem->st, cur); @@ -1788,6 +1826,17 @@ static const int caller_saved[CALLER_SAVED_REGS] = { BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 }; +static void bpf_diag_record_caller_saved(struct bpf_verifier_env *env, + struct bpf_reg_state *regs) +{ + int i; + + for (i = 1; i < CALLER_SAVED_REGS; i++) { + bpf_diag_record_scrub(env, ®s[caller_saved[i]], + BPF_DIAG_MOD_CALLER_SAVED); + } +} + /* This helper doesn't clear reg->id */ static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm) { @@ -2241,6 +2290,7 @@ static void init_func_state(struct bpf_verifier_env *env, { state->callsite = callsite; state->frameno = frameno; + bpf_diag_init_frame(env, state); state->subprogno = subprogno; state->callback_ret_range = retval_range(0, 0); init_reg_state(env, state); @@ -2263,6 +2313,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, elem->prev_insn_idx = prev_insn_idx; elem->next = env->head; elem->log_pos = env->log.end_pos; + elem->diag_log_pos = bpf_diag_event_log_save(env); env->head = elem; env->stack_size++; if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { @@ -2624,6 +2675,26 @@ static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) return btf_vmlinux ?: ERR_PTR(-ENOENT); } +static struct btf *find_kfunc_desc_btf_cached(struct bpf_verifier_env *env, s16 offset) +{ + struct bpf_kfunc_btf kf_btf = { .offset = offset }; + struct bpf_kfunc_btf_tab *tab; + struct bpf_kfunc_btf *b; + + if (!offset) + return btf_vmlinux ?: ERR_PTR(-ENOENT); + if (offset < 0) + return ERR_PTR(-EINVAL); + + tab = env->prog->aux->kfunc_btf_tab; + if (!tab) + return ERR_PTR(-ENOENT); + + b = bsearch(&kf_btf, tab->descs, tab->nr_descs, + sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); + return b ? b->btf : ERR_PTR(-ENOENT); +} + #define KF_IMPL_SUFFIX "_impl" static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log, @@ -2853,6 +2924,10 @@ static int add_subprogs(struct bpf_verifier_env *env) if (!env->bpf_capable) { verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + bpf_diag_policy( + env, i, "BPF-to-BPF function call", + "loading or calling other BPF functions requires CAP_BPF or CAP_SYS_ADMIN", + "Load this program with the required capability, or avoid BPF-to-BPF function calls in unprivileged programs."); return -EPERM; } @@ -2905,6 +2980,10 @@ static int add_kfuncs(struct bpf_verifier_env *env) if (!env->bpf_capable) { verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n"); + bpf_diag_policy( + env, i, "kernel function call", + "calling kernel functions requires CAP_BPF or CAP_SYS_ADMIN", + "Load this program with the required capability, or avoid kernel function calls in unprivileged programs."); return -EPERM; } @@ -2949,6 +3028,12 @@ static int check_subprogs(struct bpf_verifier_env *env) off = i + bpf_jmp_offset(&insn[i]) + 1; if (off < subprog_start || off >= subprog_end) { verbose(env, "jump out of range from insn %d to %d\n", i, off); + bpf_diag_program_structure( + env, i, "jump out of range", + "Keep branch targets within the same subprogram, or use an explicit subprogram call.", + "Instruction %d jumps to instruction %d, but subprogram %d only contains instructions %d through %d. " + "A branch target must stay inside the same subprogram.", + i, off, cur_subprog, subprog_start, subprog_end - 1); return -EINVAL; } next: @@ -2961,6 +3046,11 @@ static int check_subprogs(struct bpf_verifier_env *env) code != (BPF_JMP32 | BPF_JA) && code != (BPF_JMP | BPF_JA)) { verbose(env, "last insn is not an exit or jmp\n"); + bpf_diag_program_structure( + env, i, "subprogram can fall through", + "End each subprogram with an exit or an explicit jump that keeps control flow inside the subprogram.", + "Subprogram %d reaches its last instruction %d without an exit or jump, so control could continue into the next subprogram.", + cur_subprog, i); return -EINVAL; } subprog_start = subprog_end; @@ -3031,8 +3121,13 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (bpf_pseudo_func(&insn[idx])) continue; verbose(env, "recursive call from %s() to %s()\n", - subprog_name(env, cur), - subprog_name(env, callee)); + bpf_subprog_name(env, cur), + bpf_subprog_name(env, callee)); + bpf_diag_program_structure( + env, idx, "recursive subprogram call", + "Rewrite the recursion as an explicit bounded loop, or split the logic so subprogram calls do not form a cycle.", + "This bpf2bpf call would make the subprogram call graph recursive. " + "The verifier requires a finite, acyclic call graph so it can bound stack depth and analysis."); ret = -EINVAL; goto out; } @@ -3053,7 +3148,7 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_LEVEL2) for (i = 0; i < cnt; i++) verbose(env, "topo_order[%d] = %s\n", - i, subprog_name(env, env->subprog_topo_order[i])); + i, bpf_subprog_name(env, env->subprog_topo_order[i])); out: kvfree(dfs_stack); kvfree(color); @@ -3081,6 +3176,7 @@ static int __check_reg_arg(struct bpf_verifier_env *env, struct bpf_reg_state *r /* check whether register used as source operand can be read */ if (reg->type == NOT_INIT) { verbose(env, "R%d !read_ok\n", regno); + bpf_diag_unreadable_reg(env, env->insn_idx, regno); return -EACCES; } /* We don't need to worry about FP liveness because it's read-only */ @@ -3197,7 +3293,7 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) } } -static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn) { const struct btf_type *func; struct btf *desc_btf; @@ -3205,18 +3301,20 @@ static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) return NULL; - desc_btf = find_kfunc_desc_btf(data, insn->off); + desc_btf = find_kfunc_desc_btf_cached(data, insn->off); if (IS_ERR(desc_btf)) return ""; func = btf_type_by_id(desc_btf, insn->imm); + if (!func || !btf_type_is_func(func)) + return ""; return btf_name_by_offset(desc_btf, func->name_off); } void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) { const struct bpf_insn_cbs cbs = { - .cb_call = disasm_kfunc_name, + .cb_call = bpf_disasm_kfunc_name, .cb_print = verbose, .private_data = env, }; @@ -3335,6 +3433,7 @@ static void save_register_state(struct bpf_verifier_env *env, { int i; + bpf_diag_mod_begin(env, &state->stack[spi].spilled_ptr, reg, BPF_DIAG_MOD_SPILL); state->stack[spi].spilled_ptr = *reg; for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--) @@ -3343,6 +3442,8 @@ static void save_register_state(struct bpf_verifier_env *env, /* size < 8 bytes spill */ for (; i; i--) mark_stack_slot_misc(env, &state->stack[spi].slot_type[i - 1]); + + bpf_diag_mod_end(env); } static bool is_bpf_st_mem(struct bpf_insn *insn) @@ -3415,7 +3516,16 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, bpf_is_spilled_reg(&state->stack[spi]) && !bpf_is_spilled_scalar_reg(&state->stack[spi]) && size != BPF_REG_SIZE) { + const char *reason; + verbose(env, "attempt to corrupt spilled pointer on stack\n"); + reason = bpf_diag_fmt(env, + "This store writes %d bytes at stack offset %d into a stack slot that currently holds a spilled pointer. " + "Partial writes to spilled pointers are rejected because they can corrupt pointer metadata and leak kernel pointers.", + size, off); + bpf_diag_memory( + env, insn_idx, "stack spill corruption", reason, + "Write the full 8-byte spilled pointer slot, or use a separate stack slot for scalar data before overwriting only part of it."); return -EACCES; } @@ -3479,6 +3589,9 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; + if (bpf_is_spilled_reg(&state->stack[spi])) + bpf_diag_record_scrub(env, &state->stack[spi].spilled_ptr, + BPF_DIAG_MOD_WRITE); scrub_special_slot(state, spi); /* when we zero initialize stack slots mark them as such */ @@ -3639,6 +3752,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, if (err) return err; } + bpf_diag_record_scrub_stack(env, state, min_off, max_off, + BPF_DIAG_MOD_VAR_WRITE); return 0; } @@ -3702,6 +3817,21 @@ static int mark_reg_stack_read(struct bpf_verifier_env *env, return 0; } +static void bpf_diag_stack_read_uninit(struct bpf_verifier_env *env, int off, int i, + int size) +{ + const char *reason; + + reason = bpf_diag_fmt(env, + "This rejected read uses %d bytes at stack offset %d, but byte %d in that range is uninitialized on this path. " + "Programs loaded with CAP_PERFMON can be allowed to read uninitialized stack bytes, but this program is being rejected without that allowance.", + size, off, i); + bpf_diag_memory( + env, env->insn_idx, "uninitialized stack read", reason, + "Initialize every byte in the stack range before reading it, adjust the offset and size so the read covers only initialized bytes, " + "or load with CAP_PERFMON if uninitialized stack reads are intended."); +} + /* Read the stack at 'off' and put the results into the register indicated by * 'dst_regno'. It handles reg filling if the addressed stack slot is a * spilled reg. @@ -3731,6 +3861,12 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, mark_stack_slot_scratched(env, spi); check_fastcall_stack_contract(env, state, env->insn_idx, off); + /* + * Refine the in-progress load record's origin to the source stack slot. + */ + if (dst_regno >= 0) + bpf_diag_mod_begin(env, &state->regs[dst_regno], reg, BPF_DIAG_MOD_WRITE); + if (bpf_is_spilled_reg(®_state->stack[spi])) { u8 spill_size = 1; @@ -3785,6 +3921,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); + bpf_diag_stack_read_uninit(env, off, i, size); } return -EACCES; } @@ -3843,6 +3980,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, } else { verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size); + bpf_diag_stack_read_uninit(env, off, i, size); } return -EACCES; } @@ -3935,11 +4073,19 @@ static int check_stack_read(struct bpf_verifier_env *env, * check_stack_read_fixed_off). */ if (dst_regno < 0 && var_off) { + const char *reason; char tn_buf[48]; tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n", tn_buf, off, size); + reason = bpf_diag_fmt(env, + "The helper would access the stack through variable offset %s plus fixed offset %d and size %d. " + "Helper stack memory arguments require a constant stack offset and a precise initialized range.", + tn_buf, off, size); + bpf_diag_memory( + env, env->insn_idx, "variable stack access", reason, + "Use a fixed stack offset for helper memory arguments, or copy the needed bytes into a fixed stack slot first."); return -EACCES; } /* Variable offset is prohibited for unprivileged mode for simplicity @@ -4024,14 +4170,17 @@ static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_s if (spi + 1 > subprog->max_out_stack_arg_cnt) subprog->max_out_stack_arg_cnt = spi + 1; + arg = &state->stack_arg_regs[spi]; + bpf_diag_mod_begin(env, arg, value_reg, BPF_DIAG_MOD_WRITE); + if (value_reg) { state->stack_arg_regs[spi] = *value_reg; } else { /* BPF_ST: store immediate, treat as scalar */ - arg = &state->stack_arg_regs[spi]; arg->type = SCALAR_VALUE; __mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm); } + bpf_diag_mod_end(env); state->no_stack_arg_load = true; return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0); @@ -4064,7 +4213,9 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st caller = vstate->frame[vstate->curframe - 1]; arg = &caller->stack_arg_regs[spi]; cur = vstate->frame[vstate->curframe]; + bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE); cur->regs[dst_regno] = *arg; + bpf_diag_mod_end(env); return bpf_push_jmp_history(env, env->cur_state, INSN_F_STACK_ARG_ACCESS, spi, 0, 0); } @@ -4079,7 +4230,8 @@ static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx) } static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, - int nargs) + int nargs, const char *callee_name, const struct btf *btf, + const struct btf_param *args) { int i, spi; @@ -4087,8 +4239,14 @@ static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_fu spi = i - MAX_BPF_FUNC_REG_ARGS; if (spi >= caller->out_stack_arg_cnt || caller->stack_arg_regs[spi].type == NOT_INIT) { + const char *arg_name = NULL; + + if (args && args[i].name_off) + arg_name = btf_name_by_offset(btf, args[i].name_off); verbose(env, "callee expects %d args, stack arg%d is not initialized\n", nargs, spi + 1); + bpf_diag_stack_arg_uninit(env, env->insn_idx, nargs, spi, + callee_name, arg_name); return -EFAULT; } } @@ -4169,6 +4327,9 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ int off, int size, u32 mem_size, bool zero_size_allowed) { + const char *proof = ""; + const char *start; + s64 max_start, max_end; int err; /* We may have adjusted the register pointing to memory region, so we @@ -4187,14 +4348,28 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ reg_smin(reg) + off < 0)) { verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n", reg_arg_name(env, argno)); - return -EACCES; + err = -EACCES; + if (bpf_diag_enabled(env)) { + start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); + proof = bpf_diag_fmt( + env, "the minimal bound for a memory access is a negative value: %s", + start); + } + goto report_error; } + err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s min value is outside of the allowed memory range\n", reg_arg_name(env, argno)); - return err; + if (bpf_diag_enabled(env)) { + start = bpf_diag_fmt_s64_sum(env, reg_smin(reg), off); + proof = bpf_diag_fmt( + env, "the minimal bound for a memory access is %s and is outside of the object of size %u", + start, mem_size); + } + goto report_error; } /* If we haven't set a max value then we need to bail since we can't be @@ -4204,17 +4379,36 @@ static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_ if (reg_umax(reg) >= BPF_MAX_VAR_OFF) { verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n", reg_arg_name(env, argno)); - return -EACCES; + err = -EACCES; + if (bpf_diag_enabled(env)) + proof = bpf_diag_fmt( + env, "the maximal bound for a memory access is %llu and exceeds maximum allowed offset of %u", + reg_umax(reg), BPF_MAX_VAR_OFF); + goto report_error; } + err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size, mem_size, zero_size_allowed); if (err) { verbose(env, "%s max value is outside of the allowed memory range\n", reg_arg_name(env, argno)); - return err; + if (bpf_diag_enabled(env)) { + max_start = (s64)reg_umax(reg) + off; + max_end = max_start + size; + proof = bpf_diag_fmt( + env, "the maximal bound for a memory access is %lld: start %lld + access_size %d, beyond object_size %u", + max_end, max_start, size, mem_size); + } + goto report_error; } return 0; + +report_error: + bpf_diag_mem_bounds(env, env->insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg_type_str(env, reg->type), proof, + off, size, mem_size, reg); + return err; } static int __check_ptr_off_reg(struct bpf_verifier_env *env, @@ -4243,6 +4437,9 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env, if (!fixed_off_ok && reg->var_off.value != 0) { verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n", reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value); + bpf_diag_invalid_deref(env, env->insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, + BPF_DIAG_DEREF_MODIFIED_PTR, reg->var_off.value); return -EACCES; } @@ -6188,6 +6385,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (type_may_be_null(reg->type)) { verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, + BPF_DIAG_DEREF_NULLABLE_PTR, 0); return -EACCES; } @@ -6338,8 +6538,16 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b if (t == BPF_READ && value_regno >= 0) mark_reg_unknown(env, regs, value_regno); } else { + enum bpf_diag_invalid_deref_kind kind = BPF_DIAG_DEREF_INVALID_PTR; + verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + if (reg->type == SCALAR_VALUE) + kind = BPF_DIAG_DEREF_SCALAR; + else if (type_may_be_null(reg->type)) + kind = BPF_DIAG_DEREF_NULLABLE_PTR; + bpf_diag_invalid_deref(env, insn_idx, reg_from_argno(argno), + reg_arg_name(env, argno), reg, kind, 0); return -EACCES; } @@ -6399,15 +6607,19 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, src_reg_type = regs[insn->src_reg].type; - /* Check if (src_reg + off) is readable. The state of dst_reg will be - * updated by this call. + /* + * check_stack_read_fixed_off() may refine the modification's origin to + * the source stack slot. */ + bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); err = err ?: save_aux_ptr_type(env, src_reg_type, allow_trust_mismatch); err = err ?: reg_bounds_sanity_check(env, ®s[insn->dst_reg], ctx); + if (!err) + bpf_diag_mod_end(env); return err; } @@ -6513,10 +6725,14 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, */ err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true, false); - if (!err && load_reg >= 0) + if (!err && load_reg >= 0) { + bpf_diag_mod_begin(env, cur_regs(env) + load_reg, NULL, BPF_DIAG_MOD_WRITE); err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_READ, load_reg, true, false); + if (!err) + bpf_diag_mod_end(env); + } if (err) return err; @@ -6734,14 +6950,17 @@ static int check_stack_range_initialized( return 0; } -static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - int access_size, enum bpf_access_type access_type, - bool zero_size_allowed, - struct bpf_call_arg_meta *meta) +static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + argno_t argno, int access_size, + enum bpf_access_type access_type, bool zero_size_allowed, + struct bpf_call_arg_meta *meta, bool *known_memory) { struct bpf_reg_state *regs = cur_regs(env); u32 *max_access; + if (known_memory) + *known_memory = true; + switch (base_type(reg->type)) { case PTR_TO_PACKET: case PTR_TO_PACKET_META: @@ -6811,6 +7030,8 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ if (zero_size_allowed && access_size == 0 && bpf_register_is_null(reg)) return 0; + if (known_memory && base_type(reg->type) != PTR_TO_CTX) + *known_memory = false; verbose(env, "%s type=%s ", reg_arg_name(env, argno), reg_type_str(env, reg->type)); @@ -6819,6 +7040,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_ } } +enum bpf_mem_size_failure { + BPF_MEM_SIZE_FAIL_NONE, + BPF_MEM_SIZE_FAIL_MEMORY, + BPF_MEM_SIZE_FAIL_SIZE, +}; + /* verify arguments to helpers or kfuncs consisting of a pointer and an access * size. * @@ -6829,10 +7056,14 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno, u32 access_type, bool zero_size_allowed, - struct bpf_call_arg_meta *meta) + struct bpf_call_arg_meta *meta, + enum bpf_mem_size_failure *failure) { int err = 0; + if (failure) + *failure = BPF_MEM_SIZE_FAIL_NONE; + /* This is used to refine r0 return value bounds for helpers * that enforce this value as an upper bound on return values. * See do_refine_retval_range() for helpers that can refine @@ -6854,27 +7085,32 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, if (reg_smin(size_reg) < 0) { verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n", reg_arg_name(env, size_argno)); - return -EACCES; + err = -EACCES; + goto size_error; } if (reg_umin(size_reg) == 0 && !zero_size_allowed) { verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n", reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg)); - return -EACCES; + err = -EACCES; + goto size_error; } if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) { verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n", reg_arg_name(env, size_argno)); - return -EACCES; + err = -EACCES; + goto size_error; } if (access_type & BPF_READ) err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - BPF_READ, zero_size_allowed, meta); + BPF_READ, zero_size_allowed, meta, NULL); if (!err && access_type & BPF_WRITE) err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg), - BPF_WRITE, zero_size_allowed, meta); + BPF_WRITE, zero_size_allowed, meta, NULL); + if (err && failure) + *failure = BPF_MEM_SIZE_FAIL_MEMORY; if (!err) { int regno = reg_from_argno(size_argno); @@ -6886,16 +7122,23 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, } return err; + +size_error: + if (failure) + *failure = BPF_MEM_SIZE_FAIL_SIZE; + return err; } static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, u32 mem_size, enum bpf_access_type access_type, - struct bpf_call_arg_meta *meta) + struct bpf_call_arg_meta *meta, bool *known_memory) { int size, err = 0; if (bpf_register_is_null(reg)) return 0; + if (known_memory) + *known_memory = true; if (mem_size > S32_MAX) { verbose(env, "%s memory size %u is too large\n", @@ -6910,9 +7153,11 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg size = (!meta && base_type(reg->type) == PTR_TO_STACK) ? -(int)mem_size : mem_size; if (access_type & BPF_READ) - err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta); + err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, meta, + known_memory); if (!err && (access_type & BPF_WRITE)) - err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta); + err = check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, meta, + known_memory); return err; } @@ -7013,6 +7258,7 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK; const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin"; struct bpf_verifier_state *cur = env->cur_state; + struct bpf_reference_state *lock; bool is_const = tnum_is_const(reg->var_off); bool is_irq = flags & PROCESS_LOCK_IRQ; u64 val = reg->var_off.value; @@ -7062,14 +7308,25 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state ptr = btf; if (!is_res_lock && cur->active_locks) { - if (find_lock_state(env->cur_state, REF_TYPE_LOCK, 0, NULL)) { + lock = find_lock_state(cur, REF_TYPE_LOCK, 0, NULL); + if (lock) { verbose(env, "Locking two bpf_spin_locks are not allowed\n"); + bpf_diag_lock( + env, env->insn_idx, "nested spin lock", + "This path already holds a bpf_spin_lock. The verifier allows only one regular BPF spin lock at a time.", + "Unlock the current bpf_spin_lock before taking another one.", lock); return -EINVAL; } } else if (is_res_lock && cur->active_locks) { - if (find_lock_state(env->cur_state, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, reg->id, ptr)) { + lock = find_lock_state(cur, REF_TYPE_RES_LOCK | REF_TYPE_RES_LOCK_IRQ, + reg->id, ptr); + if (lock) { verbose(env, "Acquiring the same lock again, AA deadlock detected\n"); + bpf_diag_lock( + env, env->insn_idx, "recursive resource spin lock", + "This path already holds the same resource spin lock. Taking it again would deadlock.", + "Avoid reacquiring the same resource spin lock before it is unlocked.", lock); return -EINVAL; } } @@ -7096,6 +7353,10 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state if (!cur->active_locks) { verbose(env, "%s_unlock without taking a lock\n", lock_str); + bpf_diag_res( + env, env->insn_idx, "unlock without lock", + "This unlock operation has no matching active lock on the current path.", + "Take the matching lock before this unlock, or remove the unmatched unlock path."); return -EINVAL; } @@ -7105,16 +7366,35 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state type = REF_TYPE_RES_LOCK; else type = REF_TYPE_LOCK; - if (!find_lock_state(cur, type, reg->id, ptr)) { + + lock = find_lock_state(cur, type, reg->id, ptr); + if (!lock) { verbose(env, "%s_unlock of different lock\n", lock_str); + lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, + cur->active_lock_ptr); + bpf_diag_lock( + env, env->insn_idx, "unlock of a different lock", + "This unlock does not match any active lock with the same tracked identity on the current path.", + "Unlock the same lock object that was most recently acquired.", lock); return -EINVAL; } if (reg->id != cur->active_lock_id || ptr != cur->active_lock_ptr) { verbose(env, "%s_unlock cannot be out of order\n", lock_str); + lock = find_lock_state(cur, REF_TYPE_LOCK_MASK, cur->active_lock_id, + cur->active_lock_ptr); + bpf_diag_lock( + env, env->insn_idx, "unlock out of order", + "Locks must be released in last-in, first-out order, but this unlock does not match the currently active lock.", + "Release nested locks in the reverse order they were acquired.", lock); return -EINVAL; } - if (release_lock_state(cur, type, reg->id, ptr)) { + if (release_lock_state(env, type, reg->id, ptr)) { verbose(env, "%s_unlock of different lock\n", lock_str); + bpf_diag_lock( + env, env->insn_idx, "unlock of a different lock", + "The verifier could not release a lock state matching this unlock operation.", + "Pass the same lock object and lock kind that were used for the matching lock operation.", + lock); return -EINVAL; } if (!in_rcu_cs(env)) @@ -7237,6 +7517,12 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, return 0; } +static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, + const char *call_name, const char *reason, const char *suggestion); +__printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, + argno_t argno, const char *call_name, + const char *suggestion, const char *fmt, ...); + /* * Validate dynptr arguments for helper, kfunc and subprog. * @@ -7261,7 +7547,8 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno, * and checked dynamically during runtime. */ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - argno_t argno, int insn_idx, enum bpf_arg_type arg_type, + argno_t argno, int insn_idx, const char *call_name, + enum bpf_arg_type arg_type, struct ref_obj_desc *ref_obj, struct bpf_dynptr_desc *dynptr) { int spi, err = 0; @@ -7270,6 +7557,11 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat verbose(env, "%s expected pointer to stack or const struct bpf_dynptr\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, call_name, + "Pass the address of a stack dynptr object, or use a const dynptr pointer returned by the verifier-supported path.", + "a dynptr argument must be a pointer to a dynptr stack slot or a verifier-provided const struct bpf_dynptr, but %s is %s", + reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -7292,6 +7584,10 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat if (!is_dynptr_reg_valid_uninit(env, reg)) { verbose(env, "Dynptr has to be an uninitialized dynptr\n"); + bpf_diag_res( + env, insn_idx, "dynptr is already initialized", + "This kfunc constructs a dynptr and requires an uninitialized dynptr stack slot, but the selected slot already holds dynptr state.", + "Use a fresh stack dynptr slot, or release/destroy the existing dynptr before reusing the slot."); return -EINVAL; } @@ -7308,21 +7604,35 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat /* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */ if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) { verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n"); + bpf_diag_res( + env, insn_idx, "const dynptr release", + "This release operation was given a const dynptr. Const dynptr values are verifier-provided views and cannot be released by the program.", + "Release only mutable dynptrs that the program initialized or reserved."); return -EINVAL; } if (!is_dynptr_reg_valid_init(env, reg)) { verbose(env, "Expected an initialized dynptr as %s\n", reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "uninitialized dynptr use", + "This operation requires an initialized dynptr, but the stack slot does not currently hold a valid dynptr on this path.", + "Initialize the dynptr on every path before this call, and avoid overwriting or releasing it before this use."); return -EINVAL; } /* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */ if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) { - verbose(env, - "Expected a dynptr of type %s as %s\n", - dynptr_type_str(arg_to_dynptr_type(arg_type)), - reg_arg_name(env, argno)); + enum bpf_dynptr_type expected_type = arg_to_dynptr_type(arg_type); + enum bpf_dynptr_type actual_type = dynptr_reg_type(env, reg); + + verbose(env, "Expected a dynptr of type %s as %s\n", + dynptr_type_str(expected_type), reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, call_name, + "Use a dynptr constructor that matches this operation, or call an operation that accepts the dynptr's current type.", + "the dynptr is initialized with backing object type %s, but this operation expects dynptr type %s", + dynptr_type_str(actual_type), dynptr_type_str(expected_type)); return -EINVAL; } @@ -7386,6 +7696,11 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (reg->type != PTR_TO_STACK) { verbose(env, "%s expected pointer to an iterator on stack\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt( + env, insn_idx, argno, meta->func_name, + "Pass the address of a stack iterator object for iterator new, next, and destroy calls.", + "iterator state must live in verifier-tracked stack memory, but %s is %s", + reg_arg_name(env, argno), bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -7399,6 +7714,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (btf_id < 0) { verbose(env, "expected valid iter pointer as %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, meta->func_name, + "the kfunc expects a recognized iterator state pointer, but this argument does not match a valid iterator type", + "Pass the exact iterator state type expected by this kfunc."); return -EINVAL; } t = btf_type_by_id(meta->btf, btf_id); @@ -7409,6 +7728,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) { verbose(env, "expected uninitialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "iterator is already initialized", + "Iterator creation requires an uninitialized iterator stack object, but this stack range already contains iterator state.", + "Use a fresh iterator stack slot, or destroy the existing iterator before reusing the slot."); return -EINVAL; } @@ -7433,9 +7756,16 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * case -EINVAL: verbose(env, "expected an initialized iter_%s as %s\n", iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno)); + bpf_diag_res( + env, insn_idx, "uninitialized iterator use", + "This iterator operation requires an initialized iterator state object, but the stack range does not contain a live iterator on this path.", + "Call the matching iterator new kfunc on every path before calling next or destroy, and do not destroy the iterator before this use."); return err; case -EPROTO: verbose(env, "expected an RCU CS when using %s\n", meta->func_name); + bpf_diag_ctx_required( + env, insn_idx, meta->func_name, BPF_DIAG_CONTEXT_RCU, + "Wrap iterator use in bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); return err; default: return err; @@ -7855,13 +8185,70 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_PTR_TO_DYNPTR] = &dynptr_types, }; +static void bpf_diag_call_arg(struct bpf_verifier_env *env, u32 insn_idx, argno_t argno, + const char *call_name, const char *reason, + const char *suggestion) +{ + int arg = arg_from_argno(argno); + int regno = reg_from_argno(argno); + int stack_slot = -1; + + if (arg < 0 && regno >= BPF_REG_1 && regno <= BPF_REG_5) + arg = regno; + if (arg > MAX_BPF_FUNC_REG_ARGS) + stack_slot = arg - MAX_BPF_FUNC_REG_ARGS - 1; + + bpf_diag_call_type(env, insn_idx, arg, regno, stack_slot, + call_name && *call_name ? call_name : "call", + reg_arg_name(env, argno), reason, suggestion); +} + +static const char *bpf_diag_arg_name(struct bpf_verifier_env *env, argno_t argno) +{ + return bpf_diag_fmt(env, "%s", reg_arg_name(env, argno)); +} + +__printf(6, 7) static void bpf_diag_call_arg_fmt(struct bpf_verifier_env *env, u32 insn_idx, + argno_t argno, const char *call_name, + const char *suggestion, const char *fmt, ...) +{ + const char *reason; + va_list args; + + va_start(args, fmt); + reason = bpf_diag_vfmt(env, fmt, args); + va_end(args); + + bpf_diag_call_arg(env, insn_idx, argno, call_name, reason, suggestion); +} + +static const char *bpf_diag_expected_reg_types(struct bpf_verifier_env *env, + const enum bpf_reg_type *types, int count) +{ + size_t len = 0, size = 1; + char *buf; + int i; + + for (i = 0; i < count; i++) + size += strlen(reg_type_str(env, types[i])) + (i ? 2 : 0); + + buf = bpf_diag_fmt_buf(env, size); + if (!buf) + return ""; + + for (i = 0; i < count; i++) + len += scnprintf(buf + len, size - len, "%s%s", i ? ", " : "", + reg_type_str(env, types[i])); + return buf; +} + static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, - enum bpf_arg_type arg_type, - const u32 *arg_btf_id, - struct bpf_call_arg_meta *meta) + enum bpf_arg_type arg_type, const u32 *arg_btf_id, + struct bpf_call_arg_meta *meta, const char *call_name) { enum bpf_reg_type expected, type = reg->type; const struct bpf_reg_types *compatible; + const char *actual, *accepted; int i, j, err; compatible = compatible_reg_types[base_type(arg_type)]; @@ -7908,6 +8295,12 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re for (j = 0; j + 1 < i; j++) verbose(env, "%s, ", reg_type_str(env, compatible->types[j])); verbose(env, "%s\n", reg_type_str(env, compatible->types[j])); + actual = bpf_diag_fmt(env, "%s", reg_type_str(env, reg->type)); + accepted = bpf_diag_expected_reg_types(env, compatible->types, i); + bpf_diag_call_arg_fmt(env, env->insn_idx, argno, call_name, + "Pass a value with one of the accepted pointer or scalar types for this call.", + "it has type %s, but this argument accepts %s", + actual, accepted); return -EACCES; found: @@ -7944,6 +8337,10 @@ static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *re (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) { verbose(env, "Possibly NULL pointer passed to helper %s\n", reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, env->insn_idx, argno, call_name, + "the pointer may be NULL, but this call requires a non-NULL pointer", + "Add a NULL check and make the call only on the non-NULL path."); return -EACCES; } @@ -8330,7 +8727,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK) arg_btf_id = fn->arg_btf_id[arg]; - err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta); + err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta, + func_id_name(meta->func_id)); if (err) return err; @@ -8343,6 +8741,10 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { verbose(env, "release helper %s expects referenced PTR_TO_BTF_ID passed to %s\n", func_id_name(meta->func_id), reg_arg_name(env, argno)); + bpf_diag_call_arg( + env, insn_idx, argno, func_id_name(meta->func_id), + "release helpers require a value that owns a live resource returned by a matching acquire helper", + "Pass the resource-owning pointer returned by the matching acquire helper, and avoid calling the release helper after ownership has already been transferred or released."); return -EINVAL; } @@ -8371,7 +8773,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, return -EFAULT; } key_size = meta->map.ptr->key_size; - err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL); + err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL, + NULL); if (err) return err; if (can_elide_value_nullness(meta->map.ptr)) { @@ -8408,7 +8811,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size, arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + false, meta, NULL); break; case ARG_PTR_TO_PERCPU_BTF_ID: if (!reg->btf_id) { @@ -8450,7 +8853,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, */ if (arg_type & MEM_FIXED_SIZE) { err = check_mem_reg(env, reg, argno_from_reg(regno), fn->arg_size[arg], - arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta); + arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ, meta, NULL); if (err) return err; if (arg_type & MEM_ALIGNED) @@ -8461,17 +8864,17 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - false, meta); + false, meta, NULL); break; case ARG_MEM_SIZE_OR_ZERO: err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1), argno, fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ, - true, meta); + true, meta, NULL); break; case ARG_PTR_TO_DYNPTR: - err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, &meta->ref_obj, - &meta->dynptr); + err = process_dynptr_func(env, reg, argno, insn_idx, func_id_name(meta->func_id), + arg_type, &meta->ref_obj, &meta->dynptr); if (err) return err; break; @@ -8918,8 +9321,10 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env) struct bpf_reg_state *reg; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ - if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) + if (reg_is_pkt_pointer_any(reg) || reg_is_dynptr_slice_pkt(reg)) { + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_PKT_DATA_CHANGE); mark_reg_invalid(env, reg); + } })); } @@ -8949,7 +9354,7 @@ static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range reg->range = AT_PKT_END; } -static int release_reference_nomark(struct bpf_verifier_state *state, int id) +static int __release_reference_nomark(struct bpf_verifier_state *state, int id) { int i; @@ -8964,6 +9369,16 @@ static int release_reference_nomark(struct bpf_verifier_state *state, int id) return -EINVAL; } +static int release_reference_nomark(struct bpf_verifier_env *env, int id) +{ + int err; + + err = __release_reference_nomark(env->cur_state, id); + if (!err) + bpf_diag_record_ref_release(env, env->insn_idx, id); + return err; +} + static int idstack_push(struct bpf_idmap *idmap, u32 id) { int i; @@ -9006,8 +9421,10 @@ static int release_reference(struct bpf_verifier_env *env, int id) if (err) return err; - if (find_reference_state(vstate, id)) - WARN_ON_ONCE(release_reference_nomark(vstate, id)); + if (find_reference_state(vstate, id)) { + err = release_reference_nomark(env, id); + WARN_ON_ONCE(err); + } while ((id = idstack_pop(idstack))) { /* @@ -9035,10 +9452,25 @@ static int release_reference(struct bpf_verifier_env *env, int id) return err; } + /* + * A dynptr occupies two stack slots that invalidate_dynptr() + * clears together. Record both scrubs before invalidating it. + */ + if (stack && stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) { + struct bpf_stack_state *dyn_stack = stack; + + if (reg->dynptr.first_slot) + dyn_stack--; + bpf_diag_record_scrub(env, &dyn_stack[0].spilled_ptr, + BPF_DIAG_MOD_REF_RELEASE); + bpf_diag_record_scrub(env, &dyn_stack[1].spilled_ptr, + BPF_DIAG_MOD_REF_RELEASE); + invalidate_dynptr(env, dyn_stack); + continue; + } + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_REF_RELEASE); if (!stack || stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL) mark_reg_invalid(env, reg); - else if (stack->slot_type[BPF_REG_SIZE - 1] == STACK_DYNPTR) - invalidate_dynptr(env, stack); })); } @@ -9051,8 +9483,10 @@ static void invalidate_non_owning_refs(struct bpf_verifier_env *env) struct bpf_reg_state *reg; bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ - if (type_is_non_owning_ref(reg->type)) + if (type_is_non_owning_ref(reg->type)) { + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_NON_OWN_REF); mark_reg_invalid(env, reg); + } })); } @@ -9065,8 +9499,10 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env) bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({ if (reg->type & MEM_RCU) { + bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL); reg->type |= PTR_UNTRUSTED; + bpf_diag_mod_end(env); } })); } @@ -9077,15 +9513,19 @@ static int ref_convert_alloc_rcu_protected(struct bpf_verifier_env *env, u32 id) struct bpf_reg_state *reg; int err; - err = release_reference_nomark(env->cur_state, id); + err = release_reference_nomark(env, id); + if (err) + return err; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ if (reg->id != id) continue; if ((reg->type & MEM_ALLOC) && (reg->type & MEM_PERCPU)) { + bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE); reg->id = 0; reg->type &= ~MEM_ALLOC; reg->type |= MEM_RCU; + bpf_diag_mod_end(env); } })); @@ -9097,6 +9537,8 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, { int i; + bpf_diag_record_caller_saved(env, regs); + /* after the call registers r0 - r5 were scratched */ for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); @@ -9104,13 +9546,15 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env, } } -static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env, +static void invalidate_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *state) { int i, nslots = state->out_stack_arg_cnt; - for (i = 0; i < nslots; i++) + for (i = 0; i < nslots; i++) { + bpf_diag_record_scrub(env, &state->stack_arg_regs[i], BPF_DIAG_MOD_CALLER_SAVED); bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]); + } } typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env, @@ -9178,20 +9622,28 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, struct bpf_func_state *caller = cur_func(env); struct bpf_verifier_log *log = &env->log; struct ref_obj_desc ref_obj = {}; + const struct btf_param *args; + const struct btf_type *func, *func_proto; u32 i; int ret, err; ret = btf_prepare_func_args(env, subprog); if (ret) { if (bpf_in_stack_arg_cnt(sub) > 0) { - err = check_outgoing_stack_args(env, caller, sub->arg_cnt); + err = check_outgoing_stack_args(env, caller, sub->arg_cnt, + bpf_subprog_name(env, subprog), + NULL, NULL); if (err) return err; } return ret; } - ret = check_outgoing_stack_args(env, caller, sub->arg_cnt); + func = btf_type_by_id(btf, env->prog->aux->func_info[subprog].type_id); + func_proto = btf_type_by_id(btf, func->type); + args = btf_params(func_proto); + ret = check_outgoing_stack_args(env, caller, sub->arg_cnt, + bpf_subprog_name(env, subprog), btf, args); if (ret) return ret; @@ -9230,7 +9682,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE); if (ret < 0) return ret; - if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL)) + if (check_mem_reg(env, reg, argno, arg->mem_size, BPF_READ | BPF_WRITE, NULL, + NULL)) return -EINVAL; if (!(arg->arg_type & PTR_MAYBE_NULL) && (type_may_be_null(reg->type) || bpf_register_is_null(reg))) { @@ -9256,7 +9709,9 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, if (ret) return ret; - ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, &ref_obj, NULL); + ret = process_dynptr_func(env, reg, argno, env->insn_idx, + bpf_subprog_name(env, subprog), arg->arg_type, + &ref_obj, NULL); if (ret) return ret; } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) { @@ -9267,7 +9722,8 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, continue; memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */ - err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta); + err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta, + bpf_subprog_name(env, subprog)); err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type); if (err) return err; @@ -9408,17 +9864,25 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); + const char *operation; + bool returns_void; if (env->cur_state->active_locks) { verbose(env, "global function calls are not allowed while holding a lock,\n" "use static function instead\n"); + operation = bpf_diag_fmt(env, "global function %s()", sub_name); + bpf_diag_ctx_active(env, *insn_idx, operation, BPF_DIAG_CONTEXT_LOCK, + "Release the lock before calling the global function, or use a static function instead."); return -EINVAL; } if (env->subprog_info[subprog].might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable global function %s() called in %s\n", sub_name, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable global function %s()", sub_name); + bpf_diag_ctx_forbidden(env, *insn_idx, operation, + "Move the call outside the critical section, or use a non-sleepable function."); return -EINVAL; } @@ -9431,16 +9895,22 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); /* mark global subprog for verifying after main prog */ subprog_aux(env, subprog)->called = true; + if (returns_void) + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); + else + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); /* All non-void global functions return a 64-bit SCALAR_VALUE. */ - if (!subprog_returns_void(env, subprog)) { + if (!returns_void) { mark_reg_unknown(env, caller->regs, BPF_REG_0); + bpf_diag_mod_end(env); } if (env->subprog_info[subprog].might_throw) { @@ -9475,6 +9945,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; + bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); clear_caller_saved_regs(env, caller->regs); /* and go analyze first insn of the callee */ @@ -9838,7 +10309,9 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } } else { /* return to the caller whatever r0 had in the callee */ + bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); caller->regs[BPF_REG_0] = *r0; + bpf_diag_mod_end(env); } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -10044,6 +10517,7 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi continue; verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_idx); + bpf_diag_leak(env, state->refs[i].id, state->refs[i].insn_idx, env->insn_idx); refs_lingering = true; } return refs_lingering ? -EINVAL : 0; @@ -10055,6 +10529,8 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit if (check_lock && env->cur_state->active_locks) { verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_LOCK, + "Release the BPF spin lock before this operation on every path."); return -EINVAL; } @@ -10066,16 +10542,23 @@ static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit if (check_lock && env->cur_state->active_irq_id) { verbose(env, "%s cannot be used inside bpf_local_irq_save-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_IRQ, + "Restore the saved IRQ state before this operation on every path."); return -EINVAL; } if (check_lock && env->cur_state->active_rcu_locks) { verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix); + bpf_diag_ctx_active(env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_RCU, + "Call bpf_rcu_read_unlock() before this operation on every path."); return -EINVAL; } if (check_lock && env->cur_state->active_preempt_locks) { verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix); + bpf_diag_ctx_active( + env, env->insn_idx, prefix, BPF_DIAG_CONTEXT_PREEMPT, + "Call bpf_preempt_enable() before this operation on every path."); return -EINVAL; } @@ -10257,6 +10740,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn enum bpf_type_flag ret_flag; struct bpf_reg_state *regs; struct bpf_call_arg_meta meta; + const char *operation; int insn_idx = *insn_idx_p; bool changes_data; int i, err, func_id; @@ -10272,17 +10756,31 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) { verbose(env, "program of this type cannot use helper %s#%d\n", func_id_name(func_id), func_id); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, "this program type does not allow the helper", + "Use a helper allowed for this program type, or move the logic to a compatible program type."); return err; } /* eBPF programs must be GPL compatible to use GPL-ed functions */ if (!env->prog->gpl_compatible && fn->gpl_only) { verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, + "this helper is restricted to GPL-compatible programs", + "Use a GPL-compatible license, or replace the helper with one that is available to non-GPL programs."); return -EINVAL; } if (fn->allowed && !fn->allowed(env->prog)) { verbose(env, "helper call is not allowed in probe\n"); + operation = bpf_diag_fmt(env, "helper %s#%d", func_id_name(func_id), func_id); + bpf_diag_policy( + env, insn_idx, operation, + "the helper-specific policy callback rejected this program", + "Use the helper only from an allowed attach point or program configuration."); return -EINVAL; } @@ -10304,6 +10802,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (fn->might_sleep && !in_sleepable_context(env)) { verbose(env, "sleepable helper %s#%d in %s\n", func_id_name(func_id), func_id, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable helper %s#%d", + func_id_name(func_id), func_id); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Move the helper call outside the critical section, or use a non-sleepable helper."); return -EINVAL; } @@ -10491,12 +10993,14 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return err; /* reset caller saved regs */ + bpf_diag_record_caller_saved(env, regs); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); } invalidate_outgoing_stack_args(env, cur_func(env)); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); /* update return register (already marked as written above) */ ret_type = fn->ret_type; ret_flag = type_flag(ret_type); @@ -10645,6 +11149,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + bpf_diag_mod_end(env); + /* * In order for a release of any of the original or cast pointers * to invalidate all other pointers, reuse the same reference id for @@ -10661,6 +11167,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn __mark_reg_known_zero(r0); r0->type = SCALAR_VALUE; + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); regs[BPF_REG_0].type &= ~PTR_MAYBE_NULL; regs[BPF_REG_0].id = meta.ref_obj.id; } else if (is_acquire_function(func_id, meta.map.ptr)) { @@ -10679,6 +11186,8 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (err) return err; + bpf_diag_mod_end(env); + err = check_map_func_compatibility(env, meta.map.ptr, func_id); if (err) return err; @@ -11587,6 +12096,12 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (!is_irq_flag_reg_valid_uninit(env, reg)) { verbose(env, "expected uninitialized irq flag as %s\n", reg_arg_name(env, argno)); + bpf_diag_res(env, env->insn_idx, "IRQ flag is already initialized", + "Saving IRQ state requires an uninitialized stack slot for " + "the IRQ flag, but this slot already contains tracked IRQ " + "flag state.", + "Use a fresh stack slot for this save operation, or restore " + "the existing IRQ flag before reusing the slot."); return -EINVAL; } @@ -11603,6 +12118,11 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * if (err) { verbose(env, "expected an initialized irq flag as %s\n", reg_arg_name(env, argno)); + bpf_diag_res(env, env->insn_idx, "uninitialized IRQ flag restore", + "Restoring IRQ state requires a stack slot that was " + "initialized by a matching IRQ save operation on this path.", + "Pass the same stack slot that was previously initialized by " + "the matching IRQ save kfunc."); return err; } @@ -11647,8 +12167,10 @@ static void ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 id) { struct bpf_func_state *unused; struct bpf_reg_state *reg; + int err; - WARN_ON_ONCE(release_reference_nomark(env->cur_state, id)); + err = release_reference_nomark(env, id); + WARN_ON_ONCE(err); bpf_for_each_reg_in_vstate(env->cur_state, unused, reg, ({ if (reg->id == id) { @@ -12053,7 +12575,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - ret = check_outgoing_stack_args(env, caller, nargs); + ret = check_outgoing_stack_args(env, caller, nargs, func_name, btf, args); if (ret) return ret; @@ -12067,7 +12589,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me argno_t argno = argno_from_arg(i + 1); int regno = reg_from_argno(argno); bool btf_id_fixed_off_ok = true; - u32 ref_id, type_size; + u32 ref_id = args[i].type, type_size; int kf_arg_type = meta->fn->arg_type[i]; if (is_kfunc_arg_prog_aux(btf, &args[i])) { @@ -12091,28 +12613,42 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me t = btf_type_skip_modifiers(btf, args[i].type, NULL); - if (btf_type_is_ptr(t) && (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && - !type_may_be_null(kf_arg_type)) { - verbose(env, "Possibly NULL pointer passed to trusted %s\n", - reg_arg_name(env, argno)); - return -EACCES; - } - - if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && - !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { - verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", - func_name, reg_arg_name(env, argno)); - return -EINVAL; - } - - if (reg_is_referenced(env, reg)) - update_ref_obj(&meta->ref_obj, reg); - if (btf_type_is_ptr(t)) { ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); } + if (btf_type_is_ptr(t) && + (bpf_register_is_null(reg) || type_may_be_null(reg->type)) && + !type_may_be_null(kf_arg_type)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + verbose(env, "Possibly NULL pointer passed to trusted %s\n", + reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Add a NULL check and call the kfunc only on the non-NULL path.", + "the pointer may be NULL, but this kfunc requires a non-NULL pointer to %s", + expected_type); + return -EACCES; + } + + if (regno == meta->release_regno && !is_kfunc_arg_dynptr(meta->btf, &args[i]) && + !reg_is_referenced(env, reg) && !bpf_register_is_null(reg)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + verbose(env, "release kfunc %s expects referenced PTR_TO_BTF_ID passed to %s\n", + func_name, reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the resource-owning pointer returned by the matching acquire kfunc, and avoid calling the release kfunc after ownership has already been transferred or released.", + "release kfuncs require a resource-owning value of type %s returned by a matching acquire kfunc", + expected_type); + return -EINVAL; + } + + if (reg_is_referenced(env, reg)) + update_ref_obj(&meta->ref_obj, reg); if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) continue; @@ -12173,35 +12709,67 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me case KF_ARG_CONST: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = process_const_arg(env, reg, argno, meta); - if (ret < 0) + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a compile-time constant or a value the verifier can prove is constant at this call.", + "the kfunc requires this scalar argument to be a verifier-known constant, but %s is variable on this path", + reg_arg_name(env, argno)); return ret; + } break; case KF_ARG_ANYTHING: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } break; case KF_ARG_CONST_ALLOC_SIZE_OR_ZERO: if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar value for this argument, not a pointer or resource object.", + "the kfunc expects an integer scalar, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) meta->r0_rdonly = true; ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem); - if (ret < 0) + if (ret < 0) { + if (ret == -EINVAL) + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a verifier-known constant size for this kfunc buffer argument.", + "the kfunc uses this argument as a return-buffer size, but %s is invalid or variable on this path", + reg_arg_name(env, argno)); return ret; + } break; case KF_ARG_PTR_TO_CTX: if (reg->type != PTR_TO_CTX) { verbose(env, "%s expected pointer to ctx, but got %s\n", reg_arg_name(env, argno), reg_type_str(env, reg->type)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the original program context pointer or preserve it before modifying registers.", + "the kfunc expects a context pointer, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -12235,10 +12803,19 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me } else { verbose(env, "%s expected pointer to allocated object\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer returned by the matching BPF object allocation path.", + "the kfunc expects an allocated object pointer, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!reg_is_referenced(env, reg)) { verbose(env, "allocated object must be referenced\n"); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the owned object pointer before it is released or transferred.", + "the allocated object pointer in %s must still carry verifier-tracked ownership, but this pointer no longer owns a live resource", + reg_arg_name(env, argno)); return -EINVAL; } if (meta->btf == btf_vmlinux) { @@ -12275,8 +12852,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me dynptr_arg_type |= (unsigned int)get_dynptr_type_flag(parent_type); } - ret = process_dynptr_func(env, reg, argno, insn_idx, dynptr_arg_type, - &meta->ref_obj, &meta->dynptr); + ret = process_dynptr_func(env, reg, argno, insn_idx, func_name, + dynptr_arg_type, &meta->ref_obj, &meta->dynptr); if (ret < 0) return ret; break; @@ -12391,13 +12968,31 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (!is_trusted_reg(env, reg) || bpf_type_has_unsafe_modifiers(reg->type)) { if (!is_kfunc_rcu(meta)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "%s must be referenced or trusted\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer acquired from a verifier-tracked source, or call this kfunc only inside the required protection if it accepts RCU pointers.", + "the kfunc requires a trusted or resource-owning pointer to %s, but %s is %s", + expected_type, + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } if (!is_rcu_reg(reg)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "%s must be a rcu pointer\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Use this kfunc with a pointer that is valid in an RCU read lock region.", + "the kfunc requires an RCU-protected pointer to %s, but %s is %s", + expected_type, + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } } @@ -12410,6 +13005,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); + const char *expected_type; verbose(env, "%s is %s expected %s %s", reg_arg_name(env, argno), reg_type_str(env, reg->type), @@ -12417,6 +13013,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg2btf_type != NOT_INIT) verbose(env, " or %s", reg_type_str(env, reg2btf_type)); verbose(env, "\n"); + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a verifier-tracked pointer to the expected kernel object type, not a pointer to stack storage or another memory buffer.", + "the kfunc expects a pointer to %s, but this argument is %s and cannot be used as that kernel object pointer", + expected_type, + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -12428,6 +13030,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me fallthrough; case KF_ARG_PTR_TO_MEM: if (kf_arg_type & MEM_FIXED_SIZE) { + bool known_memory; + resolve_ret = btf_resolve_size(btf, ref_t, &type_size); if (IS_ERR(resolve_ret)) { verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n", @@ -12435,9 +13039,28 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me ref_tname, PTR_ERR(resolve_ret)); return -EINVAL; } - ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, meta); - if (ret < 0) + ret = check_mem_reg(env, reg, argno, type_size, BPF_READ | BPF_WRITE, + meta, &known_memory); + if (ret < 0) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); + if (known_memory) + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Pass memory with at least the required number of accessible bytes and suitable read and write access.", + "the kfunc expects %u bytes of memory for %s, but the verifier cannot prove that %s provides a readable and writable range of that size", + type_size, expected_type, + bpf_diag_reg_type_plain(env, reg->type)); + else + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Pass stack, map, context, or other verifier-known memory of the expected type and size, not an integer cast to a pointer.", + "the kfunc expects %u bytes of memory for %s, but it is %s and not verifier-known memory", + type_size, expected_type, + bpf_diag_reg_type_plain(env, reg->type)); return ret; + } } break; case KF_ARG_CONST_MEM_SIZE: @@ -12450,9 +13073,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me struct bpf_reg_state *buff_reg = get_func_arg_reg(caller, regs, i - 1); struct bpf_reg_state *size_reg = reg; argno_t buff_argno = argno_from_arg(i); + enum bpf_mem_size_failure failure; if (reg->type != SCALAR_VALUE) { verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass an integer scalar length for this memory argument.", + "the kfunc expects a scalar memory size, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } @@ -12460,11 +13089,34 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; ret = check_mem_size_reg(env, buff_reg, size_reg, buff_argno, argno, - BPF_READ | BPF_WRITE, true, meta); + BPF_READ | BPF_WRITE, true, meta, &failure); if (ret < 0) { + const char *buff_arg, *size_arg; + + buff_arg = bpf_diag_arg_name(env, buff_argno); + size_arg = bpf_diag_arg_name(env, argno); verbose(env, "%s and ", reg_arg_name(env, buff_argno)); verbose(env, "%s memory, len pair leads to invalid memory access\n", reg_arg_name(env, argno)); + if (failure == BPF_MEM_SIZE_FAIL_MEMORY) { + bpf_diag_call_arg_fmt(env, insn_idx, buff_argno, func_name, + "Pass a stack, map, context, or other verifier-known memory pointer, and keep the paired length within that object.", + "it is the memory pointer in a memory/length pair with %s, but %s does not describe verifier-readable memory for the requested length", + size_arg, buff_arg); + } else if (failure == BPF_MEM_SIZE_FAIL_SIZE) { + if (reg_smin(size_reg) < 0) + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may be negative because its signed minimum is %lld", + size_arg, reg_smin(size_reg)); + else + bpf_diag_call_arg_fmt( + env, insn_idx, argno, func_name, + "Constrain the memory size to a non-negative value smaller than BPF_MAX_VAR_SIZ before this call.", + "the memory size in %s may reach %llu bytes, but variable memory accesses must stay below %u bytes", + size_arg, reg_umax(size_reg), BPF_MAX_VAR_SIZ); + } return ret; } break; @@ -12478,8 +13130,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; case KF_ARG_PTR_TO_REFCOUNTED_KPTR: if (!type_is_ptr_alloc_obj(reg->type)) { + const char *expected_type; + + expected_type = bpf_diag_fmt_btf_type(env, btf, ref_id); verbose(env, "%s is neither owning or non-owning ref\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a pointer returned by the matching BPF object allocation or lookup operation for this kfunc.", + "the kfunc expects a pointer to BPF-managed refcounted object type %s, but this argument is not such an object pointer", + expected_type); return -EINVAL; } if (!type_is_non_owning_ref(reg->type)) @@ -12504,6 +13163,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg->type != PTR_TO_MAP_VALUE) { verbose(env, "%s doesn't point to a const string\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass a constant string pointer that the verifier recognizes, such as a string stored in a read-only map value.", + "the kfunc expects a pointer to a constant string stored in verifier-known memory, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = check_arg_const_str(env, reg, argno); @@ -12544,6 +13208,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me if (reg->type != PTR_TO_STACK) { verbose(env, "%s doesn't point to an irq flag on stack\n", reg_arg_name(env, argno)); + bpf_diag_call_arg_fmt(env, insn_idx, argno, func_name, + "Pass the same stack slot used by bpf_local_irq_save() or bpf_res_spin_lock_irqsave().", + "the kfunc expects a stack pointer to an IRQ flag slot, but %s is %s", + reg_arg_name(env, argno), + bpf_diag_reg_type_plain(env, reg->type)); return -EINVAL; } ret = process_irq_flag(env, reg, argno, meta); @@ -12984,6 +13653,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, const struct btf_type *t, *ptr_type; struct bpf_call_arg_meta meta; struct bpf_insn_aux_data *insn_aux; + const char *operation; int err, insn_idx = *insn_idx_p; u32 i, nargs, ptr_type_id; struct bpf_kfunc_desc *desc; @@ -12995,8 +13665,13 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; err = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta); - if (err == -EACCES && meta.func_name) + if (err == -EACCES && meta.func_name) { verbose(env, "calling kernel function %s is not allowed\n", meta.func_name); + operation = bpf_diag_fmt(env, "kfunc %s", meta.func_name); + bpf_diag_policy( + env, insn_idx, operation, "this program cannot call the kfunc", + "Use a kfunc allowed for this program type and attach point, or change the program context."); + } if (err) return err; desc_btf = meta.btf; @@ -13043,12 +13718,19 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (is_kfunc_destructive(&meta) && !capable(CAP_SYS_BOOT)) { verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capability\n"); + operation = bpf_diag_fmt(env, "destructive kfunc %s", meta.func_name); + bpf_diag_policy( + env, insn_idx, operation, "destructive kfuncs require CAP_SYS_BOOT", + "Load the program with CAP_SYS_BOOT, or avoid destructive kfuncs."); return -EACCES; } sleepable = bpf_is_kfunc_sleepable(&meta); if (sleepable && !in_sleepable(env)) { verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); + operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Mark the program sleepable if the program type allows it, or use a non-sleepable kfunc."); return -EACCES; } @@ -13114,22 +13796,36 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (rcu_lock) { env->cur_state->active_rcu_locks++; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, true, + env->cur_state->active_rcu_locks); } else if (rcu_unlock) { if (env->cur_state->active_rcu_locks == 0) { verbose(env, "unmatched rcu read unlock (kernel function %s)\n", func_name); + bpf_diag_ctx_underflow( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, + "Remove the extra bpf_rcu_read_unlock() call, or ensure this path first enters an RCU read lock region."); return -EINVAL; } env->cur_state->active_rcu_locks--; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_RCU, false, + env->cur_state->active_rcu_locks); if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); } else if (preempt_disable) { env->cur_state->active_preempt_locks++; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, true, + env->cur_state->active_preempt_locks); } else if (preempt_enable) { if (env->cur_state->active_preempt_locks == 0) { verbose(env, "unmatched attempt to enable preemption (kernel function %s)\n", func_name); + bpf_diag_ctx_underflow( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_PREEMPT, + "Remove the extra bpf_preempt_enable() call, or ensure this path first disables preemption."); return -EINVAL; } env->cur_state->active_preempt_locks--; + bpf_diag_record_context(env, insn_idx, BPF_DIAG_CONTEXT_PREEMPT, false, + env->cur_state->active_preempt_locks); if (!in_rcu_cs(env)) invalidate_rcu_protected_refs(env); } @@ -13137,6 +13833,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (sleepable && !in_sleepable_context(env)) { verbose(env, "kernel func %s is sleepable within %s\n", func_name, non_sleepable_context_description(env)); + operation = bpf_diag_fmt(env, "sleepable kfunc %s", func_name); + bpf_diag_ctx_forbidden(env, insn_idx, operation, + "Move the kfunc call outside the critical section, or use a non-sleepable kfunc."); return -EACCES; } @@ -13147,6 +13846,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (is_kfunc_rcu_protected(&meta) && !in_rcu_cs(env)) { verbose(env, "kernel func %s requires RCU critical section protection\n", func_name); + bpf_diag_ctx_required( + env, insn_idx, func_name, BPF_DIAG_CONTEXT_RCU, + "Call this kfunc between bpf_rcu_read_lock() and bpf_rcu_read_unlock(), keeping all exit paths balanced."); return -EACCES; } @@ -13184,6 +13886,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { u32 regno = caller_saved[i]; @@ -13335,6 +14039,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, caller_info->stack_arg_cnt = stack_arg_cnt; } + /* + * Record R0 before process_iter_next_call() snapshots the alternate + * iterator path's diagnostic position. + */ + bpf_diag_mod_end(env); + if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); if (err) @@ -13718,9 +14428,8 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, * If we return -EACCES, caller may want to try again treating pointer as a * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. */ -static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, - struct bpf_insn *insn, - const struct bpf_reg_state *ptr_reg, +static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, + u32 ptr_regno, const struct bpf_reg_state *ptr_reg, const struct bpf_reg_state *off_reg) { struct bpf_verifier_state *vstate = env->cur_state; @@ -13732,6 +14441,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_sanitize_info info = {}; u8 opcode = BPF_OP(insn->code); u32 dst = insn->dst_reg; + const char *reason; int ret, bounds_ret; dst_reg = ®s[dst]; @@ -13755,12 +14465,24 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, verbose(env, "R%d 32-bit pointer arithmetic prohibited\n", dst); + reason = bpf_diag_fmt( + env, "R%d holds %s. 32-bit ALU operations on pointers discard pointer tracking, so the verifier cannot keep the result as a safe pointer.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "32-bit pointer arithmetic", reason, + "Use a 64-bit ALU instruction with an allowed, bounded scalar offset."); return -EACCES; } if (ptr_reg->type & PTR_MAYBE_NULL) { verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", dst, reg_type_str(env, ptr_reg->type)); + reason = bpf_diag_fmt( + env, "R%d may be NULL (%s). Pointer arithmetic is allowed only after the program proves the pointer is non-NULL on this path.", + ptr_regno, reg_type_str(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer arithmetic before NULL check", reason, + "Make sure that a NULL check precedes any arithmetic performed on the pointer."); return -EACCES; } @@ -13790,6 +14512,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, default: verbose(env, "R%d pointer arithmetic on %s prohibited\n", dst, reg_type_str(env, ptr_reg->type)); + reason = bpf_diag_fmt( + env, "R%d holds %s. This pointer kind does not allow offset arithmetic.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer arithmetic is not allowed", reason, + "Do not change this pointer's offset; use it only in operations accepted for its kind."); return -EACCES; } @@ -13807,9 +14535,25 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) return 0; - if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) || - !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) + if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "The scalar offset used with R%d is unbounded or outside the verifier's safe pointer-offset range [-%u, %u].", + ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Clamp or bounds-check the scalar offset before applying it to the pointer."); return -EINVAL; + } + if (!check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "R%d already has an offset outside the verifier's safe range [-%u, %u] for %s.", + ptr_regno, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, + bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Keep the base pointer within the verifier's allowed offset range before applying more arithmetic."); + return -EINVAL; + } /* pointer types do not carry 32-bit bounds at the moment. */ __mark_reg32_unbounded(dst_reg); @@ -13852,6 +14596,13 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* scalar -= pointer. Creates an unknown scalar */ verbose(env, "R%d tried to subtract pointer from scalar\n", dst); + reason = bpf_diag_fmt( + env, "This operation subtracts pointer register R%d from scalar register R%d. " + "The verifier only tracks pointer-minus-scalar arithmetic for allowed pointer types.", + ptr_regno, dst); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer subtracted from scalar", reason, + "Keep the pointer as the base; only add or subtract bounded scalars when permitted."); return -EACCES; } /* We don't allow subtraction from FP, because (according to @@ -13861,6 +14612,12 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, if (ptr_reg->type == PTR_TO_STACK) { verbose(env, "R%d subtraction from stack pointer prohibited\n", dst); + reason = bpf_diag_fmt( + env, "R%d is a stack pointer. The verifier does not allow BPF_SUB to move stack pointers.", + ptr_regno); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "subtraction from stack pointer", reason, + "Use addition from R10 to form stack addresses within the tracked stack frame."); return -EACCES; } dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64)); @@ -13886,16 +14643,38 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, /* bitwise ops on pointers are troublesome, prohibit. */ verbose(env, "R%d bitwise operator %s on pointer prohibited\n", dst, bpf_alu_string[opcode >> 4]); + reason = bpf_diag_fmt( + env, "R%d holds %s. Bitwise operator %s would destroy the pointer value the verifier is tracking.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), + bpf_alu_string[opcode >> 4]); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "bitwise operation on pointer", reason, + "Do bitwise operations on scalar values, not on pointer-valued registers."); return -EACCES; default: /* other operators (e.g. MUL,LSH) produce non-pointer results */ verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", dst, bpf_alu_string[opcode >> 4]); + reason = bpf_diag_fmt( + env, "R%d holds %s. Operator %s is not one of the limited pointer arithmetic operations the verifier can track.", + ptr_regno, bpf_diag_reg_type_plain(env, ptr_reg->type), + bpf_alu_string[opcode >> 4]); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "invalid pointer arithmetic operator", reason, + "Use only verifier-supported addition or subtraction with a bounded scalar offset, or perform this operation on a scalar value."); return -EACCES; } - if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) + if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) { + reason = bpf_diag_fmt( + env, "After this arithmetic, R%d would be outside the verifier's safe offset range [-%u, %u] for %s.", + dst, BPF_MAX_VAR_OFF, BPF_MAX_VAR_OFF, + bpf_diag_reg_type_plain(env, ptr_reg->type)); + bpf_diag_register_type( + env, env->insn_idx, ptr_regno, "pointer offset is not safe", reason, + "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); return -EINVAL; + } reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) @@ -14867,15 +15646,15 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, if (err) return err; off_reg = *dst_reg; - return adjust_ptr_min_max_vals(env, insn, src_reg, &off_reg); + return adjust_ptr_min_max_vals(env, insn, insn->src_reg, src_reg, + &off_reg); } } else if (ptr_reg) { /* pointer += scalar */ err = mark_chain_precision(env, insn->src_reg); if (err) return err; - return adjust_ptr_min_max_vals(env, insn, - dst_reg, src_reg); + return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, dst_reg, src_reg); } else if (dst_reg->precise) { /* if dst_reg is precise, src_reg should be precise as well */ err = mark_chain_precision(env, insn->src_reg); @@ -14890,8 +15669,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, __mark_reg_known(&off_reg, insn->imm); src_reg = &off_reg; if (ptr_reg) /* pointer += K */ - return adjust_ptr_min_max_vals(env, insn, - ptr_reg, src_reg); + return adjust_ptr_min_max_vals(env, insn, insn->dst_reg, ptr_reg, src_reg); } /* Got here implies adding two SCALAR_VALUEs */ @@ -14977,6 +15755,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) u8 opcode = BPF_OP(insn->code); int err; + bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); + if (opcode == BPF_END || opcode == BPF_NEG) { /* check src operand */ err = check_reg_arg(env, insn->dst_reg, SRC_OP); @@ -15150,7 +15930,12 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; } - return reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); + err = reg_bounds_sanity_check(env, ®s[insn->dst_reg], "alu"); + if (err) + return err; + + bpf_diag_mod_end(env); + return 0; } static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, @@ -15765,7 +16550,7 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, * No one could have freed the reference state before * doing the NULL check. */ - WARN_ON_ONCE(release_reference_nomark(vstate, id)); + WARN_ON_ONCE(__release_reference_nomark(vstate, id)); bpf_for_each_reg_in_vstate(vstate, state, reg, ({ mark_ptr_or_null_reg(state, reg, id, is_null); @@ -16244,11 +17029,13 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; dst_reg = ®s[insn->dst_reg]; + bpf_diag_mod_begin(env, dst_reg, NULL, BPF_DIAG_MOD_WRITE); if (insn->src_reg == 0) { u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; dst_reg->type = SCALAR_VALUE; __mark_reg_known(®s[insn->dst_reg], imm); + bpf_diag_mod_end(env); return 0; } @@ -16272,6 +17059,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) verifier_bug(env, "pseudo btf id: unexpected dst reg type"); return -EFAULT; } + bpf_diag_mod_end(env); return 0; } @@ -16291,6 +17079,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; + bpf_diag_mod_end(env); return 0; } @@ -16301,6 +17090,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) if (map->map_type == BPF_MAP_TYPE_ARENA) { __mark_reg_unknown(env, dst_reg); dst_reg->map_ptr = map; + bpf_diag_mod_end(env); return 0; } __mark_reg_known(dst_reg, aux->map_off); @@ -16318,6 +17108,7 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EFAULT; } + bpf_diag_mod_end(env); return 0; } @@ -16396,6 +17187,8 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; /* reset caller saved regs to unreadable */ + bpf_diag_record_caller_saved(env, regs); + bpf_diag_mod_begin(env, ®s[BPF_REG_0], NULL, BPF_DIAG_MOD_WRITE); for (i = 0; i < CALLER_SAVED_REGS; i++) { bpf_mark_reg_not_init(env, ®s[caller_saved[i]]); check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); @@ -16406,6 +17199,7 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) * Already marked as written above. */ mark_reg_unknown(env, regs, BPF_REG_0); + bpf_diag_mod_end(env); /* * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 * which must be explored by the verifier when in a subprog. @@ -17332,6 +18126,10 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) !kfunc_spin_allowed(env, insn->imm, insn->off))) { verbose(env, "function calls are not allowed while holding a lock\n"); + bpf_diag_ctx_active( + env, env->insn_idx, + "function call", BPF_DIAG_CONTEXT_LOCK, + "Release the BPF spin lock before making this call, or move the call outside the locked region."); return -EINVAL; } } @@ -17416,6 +18214,27 @@ static int do_check(struct bpf_verifier_env *env) state->last_insn_idx = env->prev_insn_idx; state->insn_idx = env->insn_idx; + /* + * Record the incoming edge so active and queued paths use the same + * branch-recording path. A zero-offset conditional has identical + * successors, so its outcome cannot be reconstructed from the edge. + */ + if (!state->speculative && prev_insn_idx >= 0 && prev_insn_idx < insn_cnt) { + struct bpf_insn *prev_insn = &insns[prev_insn_idx]; + int fallthrough_idx = prev_insn_idx + 1; + int branch_idx = prev_insn_idx + bpf_jmp_offset(prev_insn) + 1; + u8 class = BPF_CLASS(prev_insn->code); + u8 opcode = BPF_OP(prev_insn->code); + + if ((class == BPF_JMP || class == BPF_JMP32) && + opcode != BPF_JA && opcode != BPF_CALL && opcode != BPF_EXIT && + opcode <= BPF_JCOND && branch_idx != fallthrough_idx) { + if (env->insn_idx == branch_idx) + bpf_diag_record_branch(env, prev_insn_idx, true); + else if (env->insn_idx == fallthrough_idx) + bpf_diag_record_branch(env, prev_insn_idx, false); + } + } if (bpf_is_prune_point(env, env->insn_idx)) { err = bpf_is_state_visited(env, env->insn_idx); @@ -18479,7 +19298,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); struct bpf_subprog_arg_info *arg; struct bpf_reg_state *reg; @@ -18591,8 +19410,11 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) ret = do_check(env); out: account_current_path(env); - if (!ret && pop_log) - bpf_vlog_reset(&env->log, 0); + if (!ret) { + if (pop_log) + bpf_vlog_reset(&env->log, 0); + bpf_diag_event_log_restore(env, 0); + } free_states(env); /* @@ -18656,7 +19478,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env) return ret; } else if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, subprog_name(env, i)); + i, bpf_subprog_name(env, i)); } /* We verified new global subprog, it might have called some @@ -20188,6 +21010,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (ret) goto err_free_env; + ret = bpf_diag_init(env); + if (ret) + goto err_prep; if (env->signature) { ret = bpf_prog_calc_tag(env->prog); if (ret < 0) @@ -20478,6 +21303,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + bpf_diag_free(env); kvfree(env); return ret; } diff --git a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c index 7918646e5bfc..d3be69a9a755 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_in_map.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_in_map.c @@ -155,6 +155,7 @@ l0_%=: r0 = 0; \ SEC("socket") __description("forgot null checking on the inner map pointer") __failure __msg("R1 type=map_ptr_or_null expected=map_ptr") +__msg("map_ptr_or_null, but this argument accepts map_ptr") __failure_unpriv __naked void on_the_inner_map_pointer(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_uninit.c b/tools/testing/selftests/bpf/progs/verifier_uninit.c index 7718cd7d19ce..691018a46049 100644 --- a/tools/testing/selftests/bpf/progs/verifier_uninit.c +++ b/tools/testing/selftests/bpf/progs/verifier_uninit.c @@ -9,6 +9,7 @@ SEC("socket") __description("read uninitialized register") __failure __msg("R2 !read_ok") +__msg("R2 has never been initialized on this path") __failure_unpriv __naked void read_uninitialized_register(void) {