diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 04cadd987169..ffa5626411ac 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4147,8 +4147,16 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off); -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump); +struct bpf_linfo_source { + const char *file; + const char *line; + u32 file_name_off; + int line_num; + int line_col; +}; + +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src); int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump); struct bpf_prog *bpf_prog_find_from_stack(void); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 27b43fda9b17..579a288bc8de 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -833,6 +833,7 @@ static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) return 0; } +struct bpf_diag; struct bpf_verifier_env; struct backtrack_state { @@ -950,6 +951,7 @@ struct bpf_verifier_env { struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; + struct bpf_diag *diag; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; @@ -1433,8 +1435,10 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, u32 frameno); u32 bpf_vlog_alignment(u32 pos); +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn); struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off); +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); diff --git a/include/linux/btf.h b/include/linux/btf.h index 3f5255d095a2..7ea13768c979 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -214,6 +214,7 @@ int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, void *obj, */ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, char *buf, int len, u64 flags); +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len); int btf_get_fd_by_id(u32 id); u32 btf_obj_id(const struct btf *btf); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 87ffde865a50..5b9d767895c9 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -8316,6 +8316,16 @@ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, return ssnprintf.len; } +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len) +{ + struct btf_show show = { + .btf = btf, + .state.type_id = type_id, + }; + + return snprintf(buf, len, "%s", btf_show_name(&show)); +} + #ifdef CONFIG_PROC_FS static void bpf_btf_show_fdinfo(struct seq_file *m, struct file *filp) { diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6a94370a2448..d55e737ed75a 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3461,24 +3461,14 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); #ifdef CONFIG_BPF_SYSCALL -void bpf_get_linfo_file_line(struct btf *btf, const struct bpf_line_info *linfo, - const char **filep, const char **linep, int *nump) +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src) { - /* Get base component of the file path. */ - if (filep) { - *filep = btf_name_by_offset(btf, linfo->file_name_off); - *filep = kbasename(*filep); - } - - /* Obtain the source line, and strip whitespace in prefix. */ - if (linep) { - *linep = btf_name_by_offset(btf, linfo->line_off); - while (isspace(**linep)) - *linep += 1; - } - - if (nump) - *nump = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->file = kbasename(btf_name_by_offset(btf, linfo->file_name_off)); + src->line = btf_name_by_offset(btf, linfo->line_off); + src->file_name_off = linfo->file_name_off; + src->line_num = BPF_LINE_INFO_LINE_NUM(linfo->line_col); + src->line_col = BPF_LINE_INFO_LINE_COL(linfo->line_col); } const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off) @@ -3521,6 +3511,7 @@ const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump) { + struct bpf_linfo_source src; int idx = -1, insn_start, insn_end, len; struct bpf_line_info *linfo; void **jited_linfo; @@ -3552,7 +3543,15 @@ int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char * if (idx == -1) return -ENOENT; - bpf_get_linfo_file_line(btf, &linfo[idx], filep, linep, nump); + bpf_get_linfo_source(btf, &linfo[idx], &src); + while (isspace(*src.line)) + src.line++; + if (filep) + *filep = src.file; + if (linep) + *linep = src.line; + if (nump) + *nump = src.line_num; return 0; } diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index e75753552a4d..815aa7938b50 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -1,12 +1,61 @@ // SPDX-License-Identifier: GPL-2.0-only // Copyright (c) 2026 Meta Platforms, Inc. and affiliates. +#include #include +#include #include +#include +#include +#include +#include #include +#include +#include "disasm.h" #include "diagnostics.h" +#define BPF_DIAG_TEXT_WIDTH 100 +#define BPF_DIAG_CONTEXT 2 +#define BPF_DIAG_CONTEXT_CNT (1 + BPF_DIAG_CONTEXT * 2) +#define BPF_DIAG_SOURCE_LANE_WIDTH 88 +#define BPF_DIAG_TAB_WIDTH 8 +#define BPF_DIAG_FMT_CHUNK_SIZE (PAGE_SIZE - sizeof(struct diag_fmt_chunk)) +#define BPF_DIAG_FMT_BUF_SIZE 256 +#define DISASM_LINE_LEN 160 + +struct disasm_line { + char text[DISASM_LINE_LEN]; + int idx; + bool valid; +}; + +struct disasm_ctx { + struct bpf_verifier_env *env; + struct seq_buf seq; +}; + +struct diag_fmt_chunk { + struct list_head node; + struct seq_buf seq; + char data[]; +}; + +struct diag_fmt_mark { + struct diag_fmt_chunk *chunk; + size_t len; +}; + +struct bpf_diag_scratch { + struct bpf_linfo_source source_lines[BPF_DIAG_CONTEXT_CNT]; + struct disasm_line disasm_lines[BPF_DIAG_CONTEXT_CNT]; +}; + +struct bpf_diag { + struct bpf_diag_scratch scratch; + struct list_head fmt_chunks; +}; + bool bpf_diag_enabled(const struct bpf_verifier_env *env) { return env->log.level & BPF_LOG_LEVEL; @@ -14,6 +63,138 @@ bool bpf_diag_enabled(const struct bpf_verifier_env *env) static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +int bpf_diag_init(struct bpf_verifier_env *env) +{ + if (!bpf_diag_enabled(env)) + return 0; + + env->diag = kzalloc_obj(struct bpf_diag, GFP_KERNEL_ACCOUNT); + if (!env->diag) + return -ENOMEM; + + INIT_LIST_HEAD(&env->diag->fmt_chunks); + return 0; +} + +static char *diag_fmt_alloc(struct bpf_verifier_env *env, size_t size) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + size_t capacity, available; + char *buf; + + if (!diag || !size || size > INT_MAX) + return NULL; + + if (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (available >= size) + goto commit; + } + + capacity = max_t(size_t, BPF_DIAG_FMT_CHUNK_SIZE, size); + chunk = kmalloc(struct_size(chunk, data, capacity), GFP_KERNEL_ACCOUNT); + if (!chunk) + return NULL; + + seq_buf_init(&chunk->seq, chunk->data, capacity); + list_add_tail(&chunk->node, &diag->fmt_chunks); + available = seq_buf_get_buf(&chunk->seq, &buf); + if (WARN_ON_ONCE(available < size)) + return NULL; + +commit: + seq_buf_commit(&chunk->seq, size); + return buf; +} + +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size) +{ + char *buf; + + buf = diag_fmt_alloc(env, size); + if (buf) + buf[0] = '\0'; + return buf; +} + +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) +{ + va_list copy; + char *buf; + int len; + + va_copy(copy, args); + len = vsnprintf(NULL, 0, fmt, copy); + va_end(copy); + if (len < 0 || len == INT_MAX) + return ""; + + buf = diag_fmt_alloc(env, len + 1); + if (buf) + vsnprintf(buf, len + 1, fmt, args); + return buf ?: ""; +} + +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) +{ + const char *buf; + va_list args; + + va_start(args, fmt); + buf = bpf_diag_vfmt(env, fmt, args); + va_end(args); + return buf; +} + +static struct diag_fmt_mark diag_fmt_save(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_mark mark = {}; + + if (!diag || list_empty(&diag->fmt_chunks)) + return mark; + + mark.chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + mark.len = mark.chunk->seq.len; + return mark; +} + +static void diag_fmt_restore(struct bpf_verifier_env *env, struct diag_fmt_mark mark) +{ + struct bpf_diag *diag = env->diag; + struct diag_fmt_chunk *chunk; + + if (!diag) + return; + + while (!list_empty(&diag->fmt_chunks)) { + chunk = list_last_entry(&diag->fmt_chunks, struct diag_fmt_chunk, node); + if (chunk == mark.chunk) + break; + list_del(&chunk->node); + kfree(chunk); + } + + if (mark.chunk) { + mark.chunk->seq.len = mark.len; + seq_buf_str(&mark.chunk->seq); + } +} + +void bpf_diag_free(struct bpf_verifier_env *env) +{ + struct bpf_diag *diag = env->diag; + + if (!diag) + return; + + diag_fmt_restore(env, (struct diag_fmt_mark){}); + kfree(diag); + env->diag = NULL; +} + static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) { va_list args; @@ -26,6 +207,179 @@ static void diag_write(struct bpf_verifier_env *env, const char *fmt, ...) va_end(args); } +static void diag_print_wrapped_prefixed(struct bpf_verifier_env *env, const char *first_prefix, + const char *next_prefix, const char *text) +{ + const char *prefix = first_prefix; + + while (*text) { + const char *line = text; + int prefix_len = strlen(prefix); + int text_width = BPF_DIAG_TEXT_WIDTH - prefix_len; + int len = 0, last_space = -1; + + if (text_width < 1) + text_width = 1; + + while (line[len] && line[len] != '\n' && len < text_width) { + if (line[len] == ' ') + last_space = len; + len++; + } + + if (line[len] && line[len] != '\n' && line[len] != ' ' && last_space > 0) + len = last_space; + + diag_write(env, "%s%.*s\n", prefix, len, line); + + text = line + len; + while (*text == ' ') + text++; + if (*text == '\n') + text++; + + prefix = next_prefix; + } +} + +static int diag_line_width(unsigned int line) +{ + int width = 1; + + while (line >= 10) { + line /= 10; + width++; + } + + return width; +} + +static int diag_line_indent(const char *line) +{ + int indent = 0; + + while (*line == ' ' || *line == '\t') { + if (*line == '\t') + indent = round_up(indent + 1, BPF_DIAG_TAB_WIDTH); + else + indent++; + line++; + } + + return indent; +} + +static void disasm_print(void *private_data, const char *fmt, ...) __printf(2, 3); + +static void disasm_print(void *private_data, const char *fmt, ...) +{ + struct disasm_ctx *ctx = private_data; + va_list args; + + va_start(args, fmt); + seq_buf_vprintf(&ctx->seq, fmt, args); + va_end(args); +} + +static const char *disasm_kfunc_name(void *private_data, const struct bpf_insn *insn) +{ + struct disasm_ctx *ctx = private_data; + + return bpf_disasm_kfunc_name(ctx->env, insn); +} + +static void format_disasm_line(struct bpf_verifier_env *env, int insn_idx, + struct disasm_line *line) +{ + struct disasm_ctx ctx = { .env = env }; + struct bpf_insn *insn; + const struct bpf_insn_cbs cbs = { + .cb_call = disasm_kfunc_name, + .cb_print = disasm_print, + .private_data = &ctx, + }; + + line->idx = insn_idx; + line->valid = false; + seq_buf_init(&ctx.seq, line->text, sizeof(line->text)); + + if (insn_idx < 0 || insn_idx >= env->prog->len) + return; + + if (insn_idx > 0 && bpf_is_ldimm64(&env->prog->insnsi[insn_idx - 1])) + return; + + insn = &env->prog->insnsi[insn_idx]; + if (bpf_is_ldimm64(insn) && insn_idx + 1 >= env->prog->len) + return; + + print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); + seq_buf_str(&ctx.seq); + ctx.seq.len = strnlen(line->text, sizeof(line->text)); + while (ctx.seq.len && line->text[ctx.seq.len - 1] == '\n') + seq_buf_pop(&ctx.seq); + seq_buf_str(&ctx.seq); + + line->valid = true; +} + +static void diag_format_source_text(char *buf, size_t size, const char *line, int width) +{ + int col = 0, len = 0; + + if (!size) + return; + if (width <= 0) { + buf[0] = '\0'; + return; + } + + line = line ?: "..."; + while (*line && col < width && len + 1 < size) { + if (*line == '\t') { + int next = round_up(col + 1, BPF_DIAG_TAB_WIDTH); + + while (col < next && col < width && len + 1 < size) { + buf[len++] = ' '; + col++; + } + line++; + continue; + } + + buf[len++] = *line++; + col++; + } + + if (*line) { + int ellipsis_len = min(3, width); + + while (len > 0 && col > width - ellipsis_len) { + len--; + col--; + } + while (ellipsis_len-- && len + 1 < size) + buf[len++] = '.'; + } + + buf[len] = '\0'; +} + +static void diag_format_source_lane(char *buf, size_t size, const char *source_prefix, + int source_line_width, int line_num, const char *line) +{ + int len, text_width; + + if (line_num <= 0) { + buf[0] = '\0'; + return; + } + + len = scnprintf(buf, size, "%s%*d | ", source_prefix, source_line_width, line_num); + text_width = BPF_DIAG_SOURCE_LANE_WIDTH - len; + diag_format_source_text(buf + len, size - len, line, text_width); +} + static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, const char *problem) { @@ -45,3 +399,139 @@ static void bpf_diag_header(struct bpf_verifier_env *env, const char *category, first = toupper(problem[0]); diag_write(env, "\nVerification failed: %s: %c%s\n", category, first, problem + 1); } + +static void diag_print_source_annotation(struct bpf_verifier_env *env, int line_width, int indent, + const char *label, const char *msg) +{ + const char *first_prefix, *next_prefix, *text; + + indent = min_t(int, indent, max_t(int, 0, BPF_DIAG_SOURCE_LANE_WIDTH - line_width - 8)); + text = bpf_diag_fmt(env, "%s: %s", label, msg); + first_prefix = bpf_diag_fmt(env, " %*s | %*s^-- ", line_width + 4, "", indent, ""); + next_prefix = bpf_diag_fmt(env, " %*s | %*s ", line_width + 4, "", indent, ""); + + diag_print_wrapped_prefixed(env, first_prefix, next_prefix, text); +} + +static void diag_print_insn_context(struct bpf_verifier_env *env, u32 insn_idx, + struct disasm_line *disasm_lines) +{ + int insn_width = diag_line_width(env->prog->len ? env->prog->len - 1 : 0); + int i; + + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + int row = i - BPF_DIAG_CONTEXT; + + format_disasm_line(env, insn_idx + row, &disasm_lines[i]); + } + + diag_write(env, " Instruction context:\n"); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + struct disasm_line *line = &disasm_lines[i]; + + if (line->valid) + diag_write(env, " %s%*d | %s\n", + line->idx == insn_idx ? ">>> " : " ", + insn_width, line->idx, line->text); + } +} + +static void bpf_diag_source(struct bpf_verifier_env *env, u32 insn_idx, const char *label, + const char *fmt, ...) +{ + struct bpf_diag_scratch *scratch; + struct bpf_linfo_source *source_lines; + struct disasm_line *disasm_lines; + struct bpf_linfo_source src = {}; + struct diag_fmt_mark mark; + const struct bpf_line_info *linfo; + const struct bpf_subprog_info *subprog; + struct btf *btf = env->prog->aux->btf; + char *source_lane; + const char *msg; + const char *func; + int start_line, end_line, width, indent, subprogno, linfo_start, linfo_end, i; + va_list args; + + if (!bpf_diag_enabled(env)) + return; + if (!env->diag) + return; + + mark = diag_fmt_save(env); + label = label ?: "note"; + scratch = &env->diag->scratch; + source_lines = scratch->source_lines; + disasm_lines = scratch->disasm_lines; + memset(source_lines, 0, sizeof(scratch->source_lines)); + memset(disasm_lines, 0, sizeof(scratch->disasm_lines)); + + va_start(args, fmt); + msg = bpf_diag_vfmt(env, fmt, args); + va_end(args); + if (!*msg) + msg = ""; + + linfo = bpf_find_linfo(env->prog, insn_idx); + if (btf && linfo) + bpf_get_linfo_source(btf, linfo, &src); + if (!src.file || !*src.file || !src.line || !*src.line) { + diag_write(env, " insn %u\n", insn_idx); + diag_print_source_annotation(env, 0, 0, label, msg); + diag_print_insn_context(env, insn_idx, disasm_lines); + goto out_restore; + } + + subprog = bpf_find_containing_subprog(env, insn_idx); + subprogno = subprog ? subprog - env->subprog_info : -ENOENT; + func = subprogno >= 0 ? bpf_subprog_name(env, subprogno) : NULL; + if (func && *func) + diag_write(env, " %s @ %s:%d:%d\n", func, src.file, src.line_num, src.line_col); + else + diag_write(env, " %s:%d:%d\n", src.file, src.line_num, src.line_col); + + start_line = src.line_num - BPF_DIAG_CONTEXT; + end_line = src.line_num + BPF_DIAG_CONTEXT; + width = diag_line_width(end_line); + indent = diag_line_indent(src.line); + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) + source_lines[i].line_num = start_line + i; + + linfo = env->prog->aux->linfo; + linfo_start = subprog ? subprog->linfo_idx : 0; + linfo_end = subprogno >= 0 && subprogno + 1 < env->subprog_cnt ? + env->subprog_info[subprogno + 1].linfo_idx : env->prog->aux->nr_linfo; + for (i = linfo_start; i < linfo_end; i++) { + struct bpf_linfo_source line_src; + int idx; + + bpf_get_linfo_source(btf, &linfo[i], &line_src); + if (line_src.file_name_off != src.file_name_off || + line_src.line_num < start_line || line_src.line_num > end_line || + !line_src.line || !*line_src.line) + continue; + + idx = line_src.line_num - start_line; + if (!source_lines[idx].line) + source_lines[idx] = line_src; + } + + diag_write(env, " Source context:\n"); + source_lane = bpf_diag_fmt_buf(env, BPF_DIAG_FMT_BUF_SIZE); + if (!source_lane) + goto out_restore; + for (i = 0; i < BPF_DIAG_CONTEXT_CNT; i++) { + const char *source_prefix; + + source_prefix = source_lines[i].line_num == src.line_num ? ">>> " : " "; + diag_format_source_lane(source_lane, BPF_DIAG_FMT_BUF_SIZE, source_prefix, width, + source_lines[i].line_num, source_lines[i].line); + diag_write(env, " %s\n", source_lane); + if (source_lines[i].line_num == src.line_num) + diag_print_source_annotation(env, width, indent, label, msg); + } + diag_print_insn_context(env, insn_idx, disasm_lines); + +out_restore: + diag_fmt_restore(env, mark); +} diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index f51aa39f0909..ba268b589ac9 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -5,10 +5,17 @@ #define __BPF_DIAGNOSTICS_H #include +#include #include struct bpf_verifier_env; bool bpf_diag_enabled(const struct bpf_verifier_env *env); +int bpf_diag_init(struct bpf_verifier_env *env); +char *bpf_diag_fmt_buf(struct bpf_verifier_env *env, size_t size); +const char *bpf_diag_vfmt(struct bpf_verifier_env *env, const char *fmt, va_list args) + __printf(2, 0); +const char *bpf_diag_fmt(struct bpf_verifier_env *env, const char *fmt, ...) __printf(2, 3); +void bpf_diag_free(struct bpf_verifier_env *env); #endif /* __BPF_DIAGNOSTICS_H */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6ac1afced20b..2f330230f8d5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -34,6 +34,7 @@ #include #include +#include "diagnostics.h" #include "disasm.h" static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { @@ -405,7 +406,7 @@ static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) return btf_type_is_void(type); } -static const char *subprog_name(const struct bpf_verifier_env *env, int subprog) +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) { struct bpf_func_info *info; @@ -2624,6 +2625,26 @@ static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset) return btf_vmlinux ?: ERR_PTR(-ENOENT); } +static struct btf *find_kfunc_desc_btf_cached(struct bpf_verifier_env *env, s16 offset) +{ + struct bpf_kfunc_btf kf_btf = { .offset = offset }; + struct bpf_kfunc_btf_tab *tab; + struct bpf_kfunc_btf *b; + + if (!offset) + return btf_vmlinux ?: ERR_PTR(-ENOENT); + if (offset < 0) + return ERR_PTR(-EINVAL); + + tab = env->prog->aux->kfunc_btf_tab; + if (!tab) + return ERR_PTR(-ENOENT); + + b = bsearch(&kf_btf, tab->descs, tab->nr_descs, + sizeof(tab->descs[0]), kfunc_btf_cmp_by_off); + return b ? b->btf : ERR_PTR(-ENOENT); +} + #define KF_IMPL_SUFFIX "_impl" static const struct btf_type *find_kfunc_impl_proto(struct bpf_verifier_log *log, @@ -3031,8 +3052,8 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (bpf_pseudo_func(&insn[idx])) continue; verbose(env, "recursive call from %s() to %s()\n", - subprog_name(env, cur), - subprog_name(env, callee)); + bpf_subprog_name(env, cur), + bpf_subprog_name(env, callee)); ret = -EINVAL; goto out; } @@ -3053,7 +3074,7 @@ static int sort_subprogs_topo(struct bpf_verifier_env *env) if (env->log.level & BPF_LOG_LEVEL2) for (i = 0; i < cnt; i++) verbose(env, "topo_order[%d] = %s\n", - i, subprog_name(env, env->subprog_topo_order[i])); + i, bpf_subprog_name(env, env->subprog_topo_order[i])); out: kvfree(dfs_stack); kvfree(color); @@ -3197,7 +3218,7 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) } } -static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn) { const struct btf_type *func; struct btf *desc_btf; @@ -3205,18 +3226,20 @@ static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn) if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL) return NULL; - desc_btf = find_kfunc_desc_btf(data, insn->off); + desc_btf = find_kfunc_desc_btf_cached(data, insn->off); if (IS_ERR(desc_btf)) return ""; func = btf_type_by_id(desc_btf, insn->imm); + if (!func || !btf_type_is_func(func)) + return ""; return btf_name_by_offset(desc_btf, func->name_off); } void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn) { const struct bpf_insn_cbs cbs = { - .cb_call = disasm_kfunc_name, + .cb_call = bpf_disasm_kfunc_name, .cb_print = verbose, .private_data = env, }; @@ -9408,7 +9431,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); if (env->cur_state->active_locks) { verbose(env, "global function calls are not allowed while holding a lock,\n" @@ -18479,7 +18502,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) regs = state->frame[state->curframe]->regs; if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) { - const char *sub_name = subprog_name(env, subprog); + const char *sub_name = bpf_subprog_name(env, subprog); struct bpf_subprog_arg_info *arg; struct bpf_reg_state *reg; @@ -18656,7 +18679,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env) return ret; } else if (env->log.level & BPF_LOG_LEVEL) { verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, subprog_name(env, i)); + i, bpf_subprog_name(env, i)); } /* We verified new global subprog, it might have called some @@ -20188,6 +20211,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size); if (ret) goto err_free_env; + ret = bpf_diag_init(env); + if (ret) + goto err_prep; if (env->signature) { ret = bpf_prog_calc_tag(env->prog); if (ret < 0) @@ -20478,6 +20504,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, kvfree(env->scc_info); kvfree(env->succ); kvfree(env->gotox_tmp_buf); + bpf_diag_free(env); kvfree(env); return ret; }