github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-01 19:13:47 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'bpf-split-verifier-c'

Browse Source 
Alexei Starovoitov says:

====================
v3->v4: Restore few minor comments and undo few function moves
v2->v3: Actually restore comments lost in patch 3
(instead of adding them to patch 4)
v1->v2: Restore comments lost in patch 3

verifier.c is huge. Split it into logically independent pieces.
No functional changes.
The diff is impossible to review over email.
'git show' shows minimal actual changes. Only plenty of moved lines.
Such split may cause backport headaches.
We should have split it long ago.
Even after split verifier.c is still 20k lines,
but further split is harder.
====================

Acked-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://patch.msgid.link/20260412152936.54262-1-alexei.starovoitov@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov 2026-04-12 12:34:31 -07:00
commit 46ffc1f782
9 changed files with 7182 additions and 7075 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

285
include/linux/bpf_verifier.h
View File

@ -279,6 +279,8 @@ static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi)
(1 << BPF_REG_3) | (1 << BPF_REG_4) | \
(1 << BPF_REG_5))
#define BPF_MAIN_FUNC (-1)
#define BPF_DYNPTR_SIZE sizeof(struct bpf_dynptr_kern)
#define BPF_DYNPTR_NR_SLOTS (BPF_DYNPTR_SIZE / BPF_REG_SIZE)
@ -983,6 +985,41 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
bpf_log(&env->log, "verifier bug: " fmt "\n", ##args); \
})
static inline void mark_prune_point(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].prune_point = true;
}
static inline bool bpf_is_prune_point(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].prune_point;
}
static inline void mark_force_checkpoint(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].force_checkpoint = true;
}
static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].force_checkpoint;
}
static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].calls_callback = true;
}
static inline bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].calls_callback;
}
static inline void mark_jmp_point(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].jmp_point = true;
}
static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
{
struct bpf_verifier_state *cur = env->cur_state;
@ -1024,6 +1061,11 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id)
*btf_id = key & 0x7FFFFFFF;
}
int bpf_check_btf_info_early(struct bpf_verifier_env *env,
const union bpf_attr *attr, bpfptr_t uattr);
int bpf_check_btf_info(struct bpf_verifier_env *env,
const union bpf_attr *attr, bpfptr_t uattr);
int bpf_check_attach_target(struct bpf_verifier_log *log,
const struct bpf_prog *prog,
const struct bpf_prog *tgt_prog,
@ -1033,6 +1075,89 @@ void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
int mark_chain_precision(struct bpf_verifier_env *env, int regno);
int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx);
int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
void bpf_clear_jmp_history(struct bpf_verifier_state *state);
int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state,
const struct bpf_verifier_state *src);
struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx);
void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self);
void bpf_free_backedges(struct bpf_scc_visit *visit);
int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
int insn_flags, u64 linked_regs);
void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist);
void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,
struct bpf_reg_state *reg);
void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg);
void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,
struct bpf_verifier_state *st);
void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
int bpf_mark_chain_precision(struct bpf_verifier_env *env,
struct bpf_verifier_state *starting_state,
int regno, bool *changed);
static inline int bpf_get_spi(s32 off)
{
return (-off - 1) / BPF_REG_SIZE;
}
static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env,
const struct bpf_reg_state *reg)
{
struct bpf_verifier_state *cur = env->cur_state;
return cur->frame[reg->frameno];
}
/* Return IP for a given frame in a call stack */
static inline u32 bpf_frame_insn_idx(struct bpf_verifier_state *st, u32 frame)
{
return frame == st->curframe
? st->insn_idx
: st->frame[frame + 1]->callsite;
}
static inline bool bpf_is_jmp_point(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].jmp_point;
}
static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack)
{
return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;
}
static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack)
{
return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE;
}
static inline bool bpf_register_is_null(struct bpf_reg_state *reg)
{
return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
}
static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
{
bt->reg_masks[frame] |= 1 << reg;
}
static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
{
bt->stack_masks[frame] |= 1ull << slot;
}
static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg)
{
return bt->reg_masks[frame] & (1 << reg);
}
static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
{
return bt->stack_masks[frame] & (1ull << slot);
}
bool bpf_map_is_rdonly(const struct bpf_map *map);
int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val,
bool is_ldsx);
@ -1179,13 +1304,91 @@ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *en
int bpf_jmp_offset(struct bpf_insn *insn);
struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);
bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog);
int bpf_find_subprog(struct bpf_verifier_env *env, int off);
int bpf_compute_const_regs(struct bpf_verifier_env *env);
int bpf_prune_dead_branches(struct bpf_verifier_env *env);
int bpf_check_cfg(struct bpf_verifier_env *env);
int bpf_compute_postorder(struct bpf_verifier_env *env);
int bpf_compute_scc(struct bpf_verifier_env *env);
struct bpf_map_desc {
struct bpf_map *ptr;
int uid;
};
struct bpf_kfunc_call_arg_meta {
/* In parameters */
struct btf *btf;
u32 func_id;
u32 kfunc_flags;
const struct btf_type *func_proto;
const char *func_name;
/* Out parameters */
u32 ref_obj_id;
u8 release_regno;
bool r0_rdonly;
u32 ret_btf_id;
u64 r0_size;
u32 subprogno;
struct {
u64 value;
bool found;
} arg_constant;
/* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling,
* generally to pass info about user-defined local kptr types to later
* verification logic
* bpf_obj_drop/bpf_percpu_obj_drop
* Record the local kptr type to be drop'd
* bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type)
* Record the local kptr type to be refcount_incr'd and use
* arg_owning_ref to determine whether refcount_acquire should be
* fallible
*/
struct btf *arg_btf;
u32 arg_btf_id;
bool arg_owning_ref;
bool arg_prog;
struct {
struct btf_field *field;
} arg_list_head;
struct {
struct btf_field *field;
} arg_rbtree_root;
struct {
enum bpf_dynptr_type type;
u32 id;
u32 ref_obj_id;
} initialized_dynptr;
struct {
u8 spi;
u8 frameno;
} iter;
struct bpf_map_desc map;
u64 mem_size;
};
int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id,
const struct bpf_func_proto **ptr);
int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id,
s16 offset, struct bpf_kfunc_call_arg_meta *meta);
bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn);
bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn);
static inline bool bpf_is_iter_next_kfunc(struct bpf_kfunc_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_ITER_NEXT;
}
static inline bool bpf_is_kfunc_sleepable(struct bpf_kfunc_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_SLEEPABLE;
}
bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta);
struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem);
int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off);
bool bpf_insn_is_cond_jump(u8 code);
bool bpf_is_may_goto_insn(struct bpf_insn *insn);
@ -1204,5 +1407,85 @@ int bpf_stack_liveness_init(struct bpf_verifier_env *env);
void bpf_stack_liveness_free(struct bpf_verifier_env *env);
int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st);
bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi);
int bpf_compute_live_registers(struct bpf_verifier_env *env);
#define BPF_MAP_KEY_POISON (1ULL << 63)
#define BPF_MAP_KEY_SEEN (1ULL << 62)
static inline bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
{
return aux->map_ptr_state.poison;
}
static inline bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
{
return aux->map_ptr_state.unpriv;
}
static inline bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
{
return aux->map_key_state & BPF_MAP_KEY_POISON;
}
static inline bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
{
return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
}
static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
{
return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
}
#define MAX_PACKET_OFF 0xffff
#define CALLER_SAVED_REGS 6
enum bpf_reg_arg_type {
SRC_OP, /* register is used as source operand */
DST_OP, /* register is used as destination operand */
DST_OP_NO_MARK /* same as above, check only, don't mark */
};
#define MAX_KFUNC_DESCS 256
struct bpf_kfunc_desc {
struct btf_func_model func_model;
u32 func_id;
s32 imm;
u16 offset;
unsigned long addr;
};
struct bpf_kfunc_desc_tab {
/* Sorted by func_id (BTF ID) and offset (fd_array offset) during
* verification. JITs do lookups by bpf_insn, where func_id may not be
* available, therefore at the end of verification do_misc_fixups()
* sorts this by imm and offset.
*/
struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
u32 nr_descs;
};
/* Functions exported from verifier.c, used by fixups.c */
bool bpf_is_reg64(struct bpf_insn *insn, u32 regno, struct bpf_reg_state *reg, enum bpf_reg_arg_type t);
void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len);
void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog);
bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env);
bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm);
int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset);
int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct bpf_insn *insn_buf, int insn_idx, int *cnt);
/* Functions in fixups.c, called from bpf_check() */
int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env);
int bpf_optimize_bpf_loop(struct bpf_verifier_env *env);
void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env);
int bpf_opt_remove_dead_code(struct bpf_verifier_env *env);
int bpf_opt_remove_nops(struct bpf_verifier_env *env);
int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, const union bpf_attr *attr);
int bpf_convert_ctx_accesses(struct bpf_verifier_env *env);
int bpf_jit_subprogs(struct bpf_verifier_env *env);
int bpf_fixup_call_args(struct bpf_verifier_env *env);
int bpf_do_misc_fixups(struct bpf_verifier_env *env);
#endif /* _LINUX_BPF_VERIFIER_H */

1
kernel/bpf/Makefile
View File

@ -11,6 +11,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
obj-$(CONFIG_BPF_SYSCALL) += fixups.o cfg.o states.o backtrack.o check_btf.o
obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o
obj-$(CONFIG_BPF_JIT) += trampoline.o

934
kernel/bpf/backtrack.c Normal file
View File

@ -0,0 +1,934 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
#include <linux/bitmap.h>
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
/* for any branch, call, exit record the history of jmps in the given state */
int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
int insn_flags, u64 linked_regs)
{
u32 cnt = cur->jmp_history_cnt;
struct bpf_jmp_history_entry *p;
size_t alloc_size;
/* combine instruction flags if we already recorded this instruction */
if (env->cur_hist_ent) {
/* atomic instructions push insn_flags twice, for READ and
* WRITE sides, but they should agree on stack slot
*/
verifier_bug_if((env->cur_hist_ent->flags & insn_flags) &&
(env->cur_hist_ent->flags & insn_flags) != insn_flags,
env, "insn history: insn_idx %d cur flags %x new flags %x",
env->insn_idx, env->cur_hist_ent->flags, insn_flags);
env->cur_hist_ent->flags |= insn_flags;
verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
"insn history: insn_idx %d linked_regs: %#llx",
env->insn_idx, env->cur_hist_ent->linked_regs);
env->cur_hist_ent->linked_regs = linked_regs;
return 0;
}
cnt++;
alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
p = krealloc(cur->jmp_history, alloc_size, GFP_KERNEL_ACCOUNT);
if (!p)
return -ENOMEM;
cur->jmp_history = p;
p = &cur->jmp_history[cnt - 1];
p->idx = env->insn_idx;
p->prev_idx = env->prev_insn_idx;
p->flags = insn_flags;
p->linked_regs = linked_regs;
cur->jmp_history_cnt = cnt;
env->cur_hist_ent = p;
return 0;
}
static bool is_atomic_load_insn(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_STX &&
BPF_MODE(insn->code) == BPF_ATOMIC &&
insn->imm == BPF_LOAD_ACQ;
}
static bool is_atomic_fetch_insn(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_STX &&
BPF_MODE(insn->code) == BPF_ATOMIC &&
(insn->imm & BPF_FETCH);
}
static int insn_stack_access_spi(int insn_flags)
{
return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
}
static int insn_stack_access_frameno(int insn_flags)
{
return insn_flags & INSN_F_FRAMENO_MASK;
}
/* Backtrack one insn at a time. If idx is not at the top of recorded
* history then previous instruction came from straight line execution.
* Return -ENOENT if we exhausted all instructions within given state.
*
* It's legal to have a bit of a looping with the same starting and ending
* insn index within the same state, e.g.: 3->4->5->3, so just because current
* instruction index is the same as state's first_idx doesn't mean we are
* done. If there is still some jump history left, we should keep going. We
* need to take into account that we might have a jump history between given
* state's parent and itself, due to checkpointing. In this case, we'll have
* history entry recording a jump from last instruction of parent state and
* first instruction of given state.
*/
static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
u32 *history)
{
u32 cnt = *history;
if (i == st->first_insn_idx) {
if (cnt == 0)
return -ENOENT;
if (cnt == 1 && st->jmp_history[0].idx == i)
return -ENOENT;
}
if (cnt && st->jmp_history[cnt - 1].idx == i) {
i = st->jmp_history[cnt - 1].prev_idx;
(*history)--;
} else {
i--;
}
return i;
}
static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st,
u32 hist_end, int insn_idx)
{
if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx)
return &st->jmp_history[hist_end - 1];
return NULL;
}
static inline void bt_init(struct backtrack_state *bt, u32 frame)
{
bt->frame = frame;
}
static inline void bt_reset(struct backtrack_state *bt)
{
struct bpf_verifier_env *env = bt->env;
memset(bt, 0, sizeof(*bt));
bt->env = env;
}
static inline u32 bt_empty(struct backtrack_state *bt)
{
u64 mask = 0;
int i;
for (i = 0; i <= bt->frame; i++)
mask |= bt->reg_masks[i] | bt->stack_masks[i];
return mask == 0;
}
static inline int bt_subprog_enter(struct backtrack_state *bt)
{
if (bt->frame == MAX_CALL_FRAMES - 1) {
verifier_bug(bt->env, "subprog enter from frame %d", bt->frame);
return -EFAULT;
}
bt->frame++;
return 0;
}
static inline int bt_subprog_exit(struct backtrack_state *bt)
{
if (bt->frame == 0) {
verifier_bug(bt->env, "subprog exit from frame 0");
return -EFAULT;
}
bt->frame--;
return 0;
}
static inline void bt_clear_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg)
{
bt->reg_masks[frame] &= ~(1 << reg);
}
static inline void bt_set_reg(struct backtrack_state *bt, u32 reg)
{
bpf_bt_set_frame_reg(bt, bt->frame, reg);
}
static inline void bt_clear_reg(struct backtrack_state *bt, u32 reg)
{
bt_clear_frame_reg(bt, bt->frame, reg);
}
static inline void bt_clear_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot)
{
bt->stack_masks[frame] &= ~(1ull << slot);
}
static inline u32 bt_frame_reg_mask(struct backtrack_state *bt, u32 frame)
{
return bt->reg_masks[frame];
}
static inline u32 bt_reg_mask(struct backtrack_state *bt)
{
return bt->reg_masks[bt->frame];
}
static inline u64 bt_frame_stack_mask(struct backtrack_state *bt, u32 frame)
{
return bt->stack_masks[frame];
}
static inline u64 bt_stack_mask(struct backtrack_state *bt)
{
return bt->stack_masks[bt->frame];
}
static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
{
return bt->reg_masks[bt->frame] & (1 << reg);
}
/* format registers bitmask, e.g., "r0,r2,r4" for 0x15 mask */
static void fmt_reg_mask(char *buf, ssize_t buf_sz, u32 reg_mask)
{
DECLARE_BITMAP(mask, 64);
bool first = true;
int i, n;
buf[0] = '\0';
bitmap_from_u64(mask, reg_mask);
for_each_set_bit(i, mask, 32) {
n = snprintf(buf, buf_sz, "%sr%d", first ? "" : ",", i);
first = false;
buf += n;
buf_sz -= n;
if (buf_sz < 0)
break;
}
}
/* format stack slots bitmask, e.g., "-8,-24,-40" for 0x15 mask */
void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask)
{
DECLARE_BITMAP(mask, 64);
bool first = true;
int i, n;
buf[0] = '\0';
bitmap_from_u64(mask, stack_mask);
for_each_set_bit(i, mask, 64) {
n = snprintf(buf, buf_sz, "%s%d", first ? "" : ",", -(i + 1) * 8);
first = false;
buf += n;
buf_sz -= n;
if (buf_sz < 0)
break;
}
}
/* For given verifier state backtrack_insn() is called from the last insn to
* the first insn. Its purpose is to compute a bitmask of registers and
* stack slots that needs precision in the parent verifier state.
*
* @idx is an index of the instruction we are currently processing;
* @subseq_idx is an index of the subsequent instruction that:
* - *would be* executed next, if jump history is viewed in forward order;
* - *was* processed previously during backtracking.
*/
static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
struct bpf_jmp_history_entry *hist, struct backtrack_state *bt)
{
struct bpf_insn *insn = env->prog->insnsi + idx;
u8 class = BPF_CLASS(insn->code);
u8 opcode = BPF_OP(insn->code);
u8 mode = BPF_MODE(insn->code);
u32 dreg = insn->dst_reg;
u32 sreg = insn->src_reg;
u32 spi, i, fr;
if (insn->code == 0)
return 0;
if (env->log.level & BPF_LOG_LEVEL2) {
fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_reg_mask(bt));
verbose(env, "mark_precise: frame%d: regs=%s ",
bt->frame, env->tmp_str_buf);
bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
verbose(env, "stack=%s before ", env->tmp_str_buf);
verbose(env, "%d: ", idx);
bpf_verbose_insn(env, insn);
}
/* If there is a history record that some registers gained range at this insn,
* propagate precision marks to those registers, so that bt_is_reg_set()
* accounts for these registers.
*/
bpf_bt_sync_linked_regs(bt, hist);
if (class == BPF_ALU || class == BPF_ALU64) {
if (!bt_is_reg_set(bt, dreg))
return 0;
if (opcode == BPF_END || opcode == BPF_NEG) {
/* sreg is reserved and unused
* dreg still need precision before this insn
*/
return 0;
} else if (opcode == BPF_MOV) {
if (BPF_SRC(insn->code) == BPF_X) {
/* dreg = sreg or dreg = (s8, s16, s32)sreg
* dreg needs precision after this insn
* sreg needs precision before this insn
*/
bt_clear_reg(bt, dreg);
if (sreg != BPF_REG_FP)
bt_set_reg(bt, sreg);
} else {
/* dreg = K
* dreg needs precision after this insn.
* Corresponding register is already marked
* as precise=true in this verifier state.
* No further markings in parent are necessary
*/
bt_clear_reg(bt, dreg);
}
} else {
if (BPF_SRC(insn->code) == BPF_X) {
/* dreg += sreg
* both dreg and sreg need precision
* before this insn
*/
if (sreg != BPF_REG_FP)
bt_set_reg(bt, sreg);
} /* else dreg += K
* dreg still needs precision before this insn
*/
}
} else if (class == BPF_LDX ||
is_atomic_load_insn(insn) ||
is_atomic_fetch_insn(insn)) {
u32 load_reg = dreg;
/*
* Atomic fetch operation writes the old value into
* a register (sreg or r0) and if it was tracked for
* precision, propagate to the stack slot like we do
* in regular ldx.
*/
if (is_atomic_fetch_insn(insn))
load_reg = insn->imm == BPF_CMPXCHG ?
BPF_REG_0 : sreg;
if (!bt_is_reg_set(bt, load_reg))
return 0;
bt_clear_reg(bt, load_reg);
/* scalars can only be spilled into stack w/o losing precision.
* Load from any other memory can be zero extended.
* The desire to keep that precision is already indicated
* by 'precise' mark in corresponding register of this state.
* No further tracking necessary.
*/
if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
return 0;
/* dreg = *(u64 *)[fp - off] was a fill from the stack.
* that [fp - off] slot contains scalar that needs to be
* tracked with precision
*/
spi = insn_stack_access_spi(hist->flags);
fr = insn_stack_access_frameno(hist->flags);
bpf_bt_set_frame_slot(bt, fr, spi);
} else if (class == BPF_STX || class == BPF_ST) {
if (bt_is_reg_set(bt, dreg))
/* stx & st shouldn't be using _scalar_ dst_reg
* to access memory. It means backtracking
* encountered a case of pointer subtraction.
*/
return -ENOTSUPP;
/* scalars can only be spilled into stack */
if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
return 0;
spi = insn_stack_access_spi(hist->flags);
fr = insn_stack_access_frameno(hist->flags);
if (!bt_is_frame_slot_set(bt, fr, spi))
return 0;
bt_clear_frame_slot(bt, fr, spi);
if (class == BPF_STX)
bt_set_reg(bt, sreg);
} else if (class == BPF_JMP || class == BPF_JMP32) {
if (bpf_pseudo_call(insn)) {
int subprog_insn_idx, subprog;
subprog_insn_idx = idx + insn->imm + 1;
subprog = bpf_find_subprog(env, subprog_insn_idx);
if (subprog < 0)
return -EFAULT;
if (bpf_subprog_is_global(env, subprog)) {
/* check that jump history doesn't have any
* extra instructions from subprog; the next
* instruction after call to global subprog
* should be literally next instruction in
* caller program
*/
verifier_bug_if(idx + 1 != subseq_idx, env,
"extra insn from subprog");
/* r1-r5 are invalidated after subprog call,
* so for global func call it shouldn't be set
* anymore
*/
if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
verifier_bug(env, "global subprog unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
/* global subprog always sets R0 */
bt_clear_reg(bt, BPF_REG_0);
return 0;
} else {
/* static subprog call instruction, which
* means that we are exiting current subprog,
* so only r1-r5 could be still requested as
* precise, r0 and r6-r10 or any stack slot in
* the current frame should be zero by now
*/
if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
verifier_bug(env, "static subprog unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
/* we are now tracking register spills correctly,
* so any instance of leftover slots is a bug
*/
if (bt_stack_mask(bt) != 0) {
verifier_bug(env,
"static subprog leftover stack slots %llx",
bt_stack_mask(bt));
return -EFAULT;
}
/* propagate r1-r5 to the caller */
for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
if (bt_is_reg_set(bt, i)) {
bt_clear_reg(bt, i);
bpf_bt_set_frame_reg(bt, bt->frame - 1, i);
}
}
if (bt_subprog_exit(bt))
return -EFAULT;
return 0;
}
} else if (bpf_is_sync_callback_calling_insn(insn) && idx != subseq_idx - 1) {
/* exit from callback subprog to callback-calling helper or
* kfunc call. Use idx/subseq_idx check to discern it from
* straight line code backtracking.
* Unlike the subprog call handling above, we shouldn't
* propagate precision of r1-r5 (if any requested), as they are
* not actually arguments passed directly to callback subprogs
*/
if (bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) {
verifier_bug(env, "callback unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
if (bt_stack_mask(bt) != 0) {
verifier_bug(env, "callback leftover stack slots %llx",
bt_stack_mask(bt));
return -EFAULT;
}
/* clear r1-r5 in callback subprog's mask */
for (i = BPF_REG_1; i <= BPF_REG_5; i++)
bt_clear_reg(bt, i);
if (bt_subprog_exit(bt))
return -EFAULT;
return 0;
} else if (opcode == BPF_CALL) {
/* kfunc with imm==0 is invalid and fixup_kfunc_call will
* catch this error later. Make backtracking conservative
* with ENOTSUPP.
*/
if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
return -ENOTSUPP;
/* regular helper call sets R0 */
bt_clear_reg(bt, BPF_REG_0);
if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
/* if backtracking was looking for registers R1-R5
* they should have been found already.
*/
verifier_bug(env, "backtracking call unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
if (insn->src_reg == BPF_REG_0 && insn->imm == BPF_FUNC_tail_call
&& subseq_idx - idx != 1) {
if (bt_subprog_enter(bt))
return -EFAULT;
}
} else if (opcode == BPF_EXIT) {
bool r0_precise;
/* Backtracking to a nested function call, 'idx' is a part of
* the inner frame 'subseq_idx' is a part of the outer frame.
* In case of a regular function call, instructions giving
* precision to registers R1-R5 should have been found already.
* In case of a callback, it is ok to have R1-R5 marked for
* backtracking, as these registers are set by the function
* invoking callback.
*/
if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))
for (i = BPF_REG_1; i <= BPF_REG_5; i++)
bt_clear_reg(bt, i);
if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
verifier_bug(env, "backtracking exit unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
/* BPF_EXIT in subprog or callback always returns
* right after the call instruction, so by checking
* whether the instruction at subseq_idx-1 is subprog
* call or not we can distinguish actual exit from
* *subprog* from exit from *callback*. In the former
* case, we need to propagate r0 precision, if
* necessary. In the former we never do that.
*/
r0_precise = subseq_idx - 1 >= 0 &&
bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
bt_is_reg_set(bt, BPF_REG_0);
bt_clear_reg(bt, BPF_REG_0);
if (bt_subprog_enter(bt))
return -EFAULT;
if (r0_precise)
bt_set_reg(bt, BPF_REG_0);
/* r6-r9 and stack slots will stay set in caller frame
* bitmasks until we return back from callee(s)
*/
return 0;
} else if (BPF_SRC(insn->code) == BPF_X) {
if (!bt_is_reg_set(bt, dreg) && !bt_is_reg_set(bt, sreg))
return 0;
/* dreg <cond> sreg
* Both dreg and sreg need precision before
* this insn. If only sreg was marked precise
* before it would be equally necessary to
* propagate it to dreg.
*/
if (!hist || !(hist->flags & INSN_F_SRC_REG_STACK))
bt_set_reg(bt, sreg);
if (!hist || !(hist->flags & INSN_F_DST_REG_STACK))
bt_set_reg(bt, dreg);
} else if (BPF_SRC(insn->code) == BPF_K) {
/* dreg <cond> K
* Only dreg still needs precision before
* this insn, so for the K-based conditional
* there is nothing new to be marked.
*/
}
} else if (class == BPF_LD) {
if (!bt_is_reg_set(bt, dreg))
return 0;
bt_clear_reg(bt, dreg);
/* It's ld_imm64 or ld_abs or ld_ind.
* For ld_imm64 no further tracking of precision
* into parent is necessary
*/
if (mode == BPF_IND || mode == BPF_ABS)
/* to be analyzed */
return -ENOTSUPP;
}
/* Propagate precision marks to linked registers, to account for
* registers marked as precise in this function.
*/
bpf_bt_sync_linked_regs(bt, hist);
return 0;
}
/* the scalar precision tracking algorithm:
* . at the start all registers have precise=false.
* . scalar ranges are tracked as normal through alu and jmp insns.
* . once precise value of the scalar register is used in:
* . ptr + scalar alu
* . if (scalar cond K|scalar)
* . helper_call(.., scalar, ...) where ARG_CONST is expected
* backtrack through the verifier states and mark all registers and
* stack slots with spilled constants that these scalar registers
* should be precise.
* . during state pruning two registers (or spilled stack slots)
* are equivalent if both are not precise.
*
* Note the verifier cannot simply walk register parentage chain,
* since many different registers and stack slots could have been
* used to compute single precise scalar.
*
* The approach of starting with precise=true for all registers and then
* backtrack to mark a register as not precise when the verifier detects
* that program doesn't care about specific value (e.g., when helper
* takes register as ARG_ANYTHING parameter) is not safe.
*
* It's ok to walk single parentage chain of the verifier states.
* It's possible that this backtracking will go all the way till 1st insn.
* All other branches will be explored for needing precision later.
*
* The backtracking needs to deal with cases like:
* R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
* r9 -= r8
* r5 = r9
* if r5 > 0x79f goto pc+7
* R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
* r5 += 1
* ...
* call bpf_perf_event_output#25
* where .arg5_type = ARG_CONST_SIZE_OR_ZERO
*
* and this case:
* r6 = 1
* call foo // uses callee's r6 inside to compute r0
* r0 += r6
* if r0 == 0 goto
*
* to track above reg_mask/stack_mask needs to be independent for each frame.
*
* Also if parent's curframe > frame where backtracking started,
* the verifier need to mark registers in both frames, otherwise callees
* may incorrectly prune callers. This is similar to
* commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
*
* For now backtracking falls back into conservative marking.
*/
void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env,
struct bpf_verifier_state *st)
{
struct bpf_func_state *func;
struct bpf_reg_state *reg;
int i, j;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "mark_precise: frame%d: falling back to forcing all scalars precise\n",
st->curframe);
}
/* big hammer: mark all scalars precise in this path.
* pop_stack may still get !precise scalars.
* We also skip current state and go straight to first parent state,
* because precision markings in current non-checkpointed state are
* not needed. See why in the comment in __mark_chain_precision below.
*/
for (st = st->parent; st; st = st->parent) {
for (i = 0; i <= st->curframe; i++) {
func = st->frame[i];
for (j = 0; j < BPF_REG_FP; j++) {
reg = &func->regs[j];
if (reg->type != SCALAR_VALUE || reg->precise)
continue;
reg->precise = true;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "force_precise: frame%d: forcing r%d to be precise\n",
i, j);
}
}
for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
if (!bpf_is_spilled_reg(&func->stack[j]))
continue;
reg = &func->stack[j].spilled_ptr;
if (reg->type != SCALAR_VALUE || reg->precise)
continue;
reg->precise = true;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "force_precise: frame%d: forcing fp%d to be precise\n",
i, -(j + 1) * 8);
}
}
}
}
}
/*
* bpf_mark_chain_precision() backtracks BPF program instruction sequence and
* chain of verifier states making sure that register *regno* (if regno >= 0)
* and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
* SCALARS, as well as any other registers and slots that contribute to
* a tracked state of given registers/stack slots, depending on specific BPF
* assembly instructions (see backtrack_insns() for exact instruction handling
* logic). This backtracking relies on recorded jmp_history and is able to
* traverse entire chain of parent states. This process ends only when all the
* necessary registers/slots and their transitive dependencies are marked as
* precise.
*
* One important and subtle aspect is that precise marks *do not matter* in
* the currently verified state (current state). It is important to understand
* why this is the case.
*
* First, note that current state is the state that is not yet "checkpointed",
* i.e., it is not yet put into env->explored_states, and it has no children
* states as well. It's ephemeral, and can end up either a) being discarded if
* compatible explored state is found at some point or BPF_EXIT instruction is
* reached or b) checkpointed and put into env->explored_states, branching out
* into one or more children states.
*
* In the former case, precise markings in current state are completely
* ignored by state comparison code (see regsafe() for details). Only
* checkpointed ("old") state precise markings are important, and if old
* state's register/slot is precise, regsafe() assumes current state's
* register/slot as precise and checks value ranges exactly and precisely. If
* states turn out to be compatible, current state's necessary precise
* markings and any required parent states' precise markings are enforced
* after the fact with propagate_precision() logic, after the fact. But it's
* important to realize that in this case, even after marking current state
* registers/slots as precise, we immediately discard current state. So what
* actually matters is any of the precise markings propagated into current
* state's parent states, which are always checkpointed (due to b) case above).
* As such, for scenario a) it doesn't matter if current state has precise
* markings set or not.
*
* Now, for the scenario b), checkpointing and forking into child(ren)
* state(s). Note that before current state gets to checkpointing step, any
* processed instruction always assumes precise SCALAR register/slot
* knowledge: if precise value or range is useful to prune jump branch, BPF
* verifier takes this opportunity enthusiastically. Similarly, when
* register's value is used to calculate offset or memory address, exact
* knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
* what we mentioned above about state comparison ignoring precise markings
* during state comparison, BPF verifier ignores and also assumes precise
* markings *at will* during instruction verification process. But as verifier
* assumes precision, it also propagates any precision dependencies across
* parent states, which are not yet finalized, so can be further restricted
* based on new knowledge gained from restrictions enforced by their children
* states. This is so that once those parent states are finalized, i.e., when
* they have no more active children state, state comparison logic in
* is_state_visited() would enforce strict and precise SCALAR ranges, if
* required for correctness.
*
* To build a bit more intuition, note also that once a state is checkpointed,
* the path we took to get to that state is not important. This is crucial
* property for state pruning. When state is checkpointed and finalized at
* some instruction index, it can be correctly and safely used to "short
* circuit" any *compatible* state that reaches exactly the same instruction
* index. I.e., if we jumped to that instruction from a completely different
* code path than original finalized state was derived from, it doesn't
* matter, current state can be discarded because from that instruction
* forward having a compatible state will ensure we will safely reach the
* exit. States describe preconditions for further exploration, but completely
* forget the history of how we got here.
*
* This also means that even if we needed precise SCALAR range to get to
* finalized state, but from that point forward *that same* SCALAR register is
* never used in a precise context (i.e., it's precise value is not needed for
* correctness), it's correct and safe to mark such register as "imprecise"
* (i.e., precise marking set to false). This is what we rely on when we do
* not set precise marking in current state. If no child state requires
* precision for any given SCALAR register, it's safe to dictate that it can
* be imprecise. If any child state does require this register to be precise,
* we'll mark it precise later retroactively during precise markings
* propagation from child state to parent states.
*
* Skipping precise marking setting in current state is a mild version of
* relying on the above observation. But we can utilize this property even
* more aggressively by proactively forgetting any precise marking in the
* current state (which we inherited from the parent state), right before we
* checkpoint it and branch off into new child state. This is done by
* mark_all_scalars_imprecise() to hopefully get more permissive and generic
* finalized states which help in short circuiting more future states.
*/
int bpf_mark_chain_precision(struct bpf_verifier_env *env,
struct bpf_verifier_state *starting_state,
int regno,
bool *changed)
{
struct bpf_verifier_state *st = starting_state;
struct backtrack_state *bt = &env->bt;
int first_idx = st->first_insn_idx;
int last_idx = starting_state->insn_idx;
int subseq_idx = -1;
struct bpf_func_state *func;
bool tmp, skip_first = true;
struct bpf_reg_state *reg;
int i, fr, err;
if (!env->bpf_capable)
return 0;
changed = changed ?: &tmp;
/* set frame number from which we are starting to backtrack */
bt_init(bt, starting_state->curframe);
/* Do sanity checks against current state of register and/or stack
* slot, but don't set precise flag in current state, as precision
* tracking in the current state is unnecessary.
*/
func = st->frame[bt->frame];
if (regno >= 0) {
reg = &func->regs[regno];
if (reg->type != SCALAR_VALUE) {
verifier_bug(env, "backtracking misuse");
return -EFAULT;
}
bt_set_reg(bt, regno);
}
if (bt_empty(bt))
return 0;
for (;;) {
DECLARE_BITMAP(mask, 64);
u32 history = st->jmp_history_cnt;
struct bpf_jmp_history_entry *hist;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
bt->frame, last_idx, first_idx, subseq_idx);
}
if (last_idx < 0) {
/* we are at the entry into subprog, which
* is expected for global funcs, but only if
* requested precise registers are R1-R5
* (which are global func's input arguments)
*/
if (st->curframe == 0 &&
st->frame[0]->subprogno > 0 &&
st->frame[0]->callsite == BPF_MAIN_FUNC &&
bt_stack_mask(bt) == 0 &&
(bt_reg_mask(bt) & ~BPF_REGMASK_ARGS) == 0) {
bitmap_from_u64(mask, bt_reg_mask(bt));
for_each_set_bit(i, mask, 32) {
reg = &st->frame[0]->regs[i];
bt_clear_reg(bt, i);
if (reg->type == SCALAR_VALUE) {
reg->precise = true;
*changed = true;
}
}
return 0;
}
verifier_bug(env, "backtracking func entry subprog %d reg_mask %x stack_mask %llx",
st->frame[0]->subprogno, bt_reg_mask(bt), bt_stack_mask(bt));
return -EFAULT;
}
for (i = last_idx;;) {
if (skip_first) {
err = 0;
skip_first = false;
} else {
hist = get_jmp_hist_entry(st, history, i);
err = backtrack_insn(env, i, subseq_idx, hist, bt);
}
if (err == -ENOTSUPP) {
bpf_mark_all_scalars_precise(env, starting_state);
bt_reset(bt);
return 0;
} else if (err) {
return err;
}
if (bt_empty(bt))
/* Found assignment(s) into tracked register in this state.
* Since this state is already marked, just return.
* Nothing to be tracked further in the parent state.
*/
return 0;
subseq_idx = i;
i = get_prev_insn_idx(st, i, &history);
if (i == -ENOENT)
break;
if (i >= env->prog->len) {
/* This can happen if backtracking reached insn 0
* and there are still reg_mask or stack_mask
* to backtrack.
* It means the backtracking missed the spot where
* particular register was initialized with a constant.
*/
verifier_bug(env, "backtracking idx %d", i);
return -EFAULT;
}
}
st = st->parent;
if (!st)
break;
for (fr = bt->frame; fr >= 0; fr--) {
func = st->frame[fr];
bitmap_from_u64(mask, bt_frame_reg_mask(bt, fr));
for_each_set_bit(i, mask, 32) {
reg = &func->regs[i];
if (reg->type != SCALAR_VALUE) {
bt_clear_frame_reg(bt, fr, i);
continue;
}
if (reg->precise) {
bt_clear_frame_reg(bt, fr, i);
} else {
reg->precise = true;
*changed = true;
}
}
bitmap_from_u64(mask, bt_frame_stack_mask(bt, fr));
for_each_set_bit(i, mask, 64) {
if (verifier_bug_if(i >= func->allocated_stack / BPF_REG_SIZE,
env, "stack slot %d, total slots %d",
i, func->allocated_stack / BPF_REG_SIZE))
return -EFAULT;
if (!bpf_is_spilled_scalar_reg(&func->stack[i])) {
bt_clear_frame_slot(bt, fr, i);
continue;
}
reg = &func->stack[i].spilled_ptr;
if (reg->precise) {
bt_clear_frame_slot(bt, fr, i);
} else {
reg->precise = true;
*changed = true;
}
}
if (env->log.level & BPF_LOG_LEVEL2) {
fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
bt_frame_reg_mask(bt, fr));
verbose(env, "mark_precise: frame%d: parent state regs=%s ",
fr, env->tmp_str_buf);
bpf_fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
bt_frame_stack_mask(bt, fr));
verbose(env, "stack=%s: ", env->tmp_str_buf);
print_verifier_state(env, st, fr, true);
}
}
if (bt_empty(bt))
return 0;
subseq_idx = first_idx;
last_idx = st->last_insn_idx;
first_idx = st->first_insn_idx;
}
/* if we still have requested precise regs or slots, we missed
* something (e.g., stack access through non-r10 register), so
* fallback to marking all precise
*/
if (!bt_empty(bt)) {
bpf_mark_all_scalars_precise(env, starting_state);
bt_reset(bt);
}
return 0;
}

872
kernel/bpf/cfg.c Normal file
View File

@ -0,0 +1,872 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
#include <linux/sort.h>
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
/* non-recursive DFS pseudo code
* 1 procedure DFS-iterative(G,v):
* 2 label v as discovered
* 3 let S be a stack
* 4 S.push(v)
* 5 while S is not empty
* 6 t <- S.peek()
* 7 if t is what we're looking for:
* 8 return t
* 9 for all edges e in G.adjacentEdges(t) do
* 10 if edge e is already labelled
* 11 continue with the next edge
* 12 w <- G.adjacentVertex(t,e)
* 13 if vertex w is not discovered and not explored
* 14 label e as tree-edge
* 15 label w as discovered
* 16 S.push(w)
* 17 continue at 5
* 18 else if vertex w is discovered
* 19 label e as back-edge
* 20 else
* 21 // vertex w is explored
* 22 label e as forward- or cross-edge
* 23 label t as explored
* 24 S.pop()
*
* convention:
* 0x10 - discovered
* 0x11 - discovered and fall-through edge labelled
* 0x12 - discovered and fall-through and branch edges labelled
* 0x20 - explored
*/
enum {
DISCOVERED = 0x10,
EXPLORED = 0x20,
FALLTHROUGH = 1,
BRANCH = 2,
};
static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off)
{
struct bpf_subprog_info *subprog;
subprog = bpf_find_containing_subprog(env, off);
subprog->changes_pkt_data = true;
}
static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
{
struct bpf_subprog_info *subprog;
subprog = bpf_find_containing_subprog(env, off);
subprog->might_sleep = true;
}
/* 't' is an index of a call-site.
* 'w' is a callee entry point.
* Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
* Rely on DFS traversal order and absence of recursive calls to guarantee that
* callee's change_pkt_data marks would be correct at that moment.
*/
static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
{
struct bpf_subprog_info *caller, *callee;
caller = bpf_find_containing_subprog(env, t);
callee = bpf_find_containing_subprog(env, w);
caller->changes_pkt_data |= callee->changes_pkt_data;
caller->might_sleep |= callee->might_sleep;
}
enum {
DONE_EXPLORING = 0,
KEEP_EXPLORING = 1,
};
/* t, w, e - match pseudo-code above:
* t - index of current instruction
* w - next instruction
* e - edge
*/
static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
{
int *insn_stack = env->cfg.insn_stack;
int *insn_state = env->cfg.insn_state;
if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
return DONE_EXPLORING;
if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
return DONE_EXPLORING;
if (w < 0 || w >= env->prog->len) {
verbose_linfo(env, t, "%d: ", t);
verbose(env, "jump out of range from insn %d to %d\n", t, w);
return -EINVAL;
}
if (e == BRANCH) {
/* mark branch target for state pruning */
mark_prune_point(env, w);
mark_jmp_point(env, w);
}
if (insn_state[w] == 0) {
/* tree-edge */
insn_state[t] = DISCOVERED | e;
insn_state[w] = DISCOVERED;
if (env->cfg.cur_stack >= env->prog->len)
return -E2BIG;
insn_stack[env->cfg.cur_stack++] = w;
return KEEP_EXPLORING;
} else if ((insn_state[w] & 0xF0) == DISCOVERED) {
if (env->bpf_capable)
return DONE_EXPLORING;
verbose_linfo(env, t, "%d: ", t);
verbose_linfo(env, w, "%d: ", w);
verbose(env, "back-edge from insn %d to %d\n", t, w);
return -EINVAL;
} else if (insn_state[w] == EXPLORED) {
/* forward- or cross-edge */
insn_state[t] = DISCOVERED | e;
} else {
verifier_bug(env, "insn state internal bug");
return -EFAULT;
}
return DONE_EXPLORING;
}
static int visit_func_call_insn(int t, struct bpf_insn *insns,
struct bpf_verifier_env *env,
bool visit_callee)
{
int ret, insn_sz;
int w;
insn_sz = bpf_is_ldimm64(&insns[t]) ? 2 : 1;
ret = push_insn(t, t + insn_sz, FALLTHROUGH, env);
if (ret)
return ret;
mark_prune_point(env, t + insn_sz);
/* when we exit from subprog, we need to record non-linear history */
mark_jmp_point(env, t + insn_sz);
if (visit_callee) {
w = t + insns[t].imm + 1;
mark_prune_point(env, t);
merge_callee_effects(env, t, w);
ret = push_insn(t, w, BRANCH, env);
}
return ret;
}
struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem)
{
size_t new_size = sizeof(struct bpf_iarray) + n_elem * sizeof(old->items[0]);
struct bpf_iarray *new;
new = kvrealloc(old, new_size, GFP_KERNEL_ACCOUNT);
if (!new) {
/* this is what callers always want, so simplify the call site */
kvfree(old);
return NULL;
}
new->cnt = n_elem;
return new;
}
static int copy_insn_array(struct bpf_map *map, u32 start, u32 end, u32 *items)
{
struct bpf_insn_array_value *value;
u32 i;
for (i = start; i <= end; i++) {
value = map->ops->map_lookup_elem(map, &i);
/*
* map_lookup_elem of an array map will never return an error,
* but not checking it makes some static analysers to worry
*/
if (IS_ERR(value))
return PTR_ERR(value);
else if (!value)
return -EINVAL;
items[i - start] = value->xlated_off;
}
return 0;
}
static int cmp_ptr_to_u32(const void *a, const void *b)
{
return *(u32 *)a - *(u32 *)b;
}
static int sort_insn_array_uniq(u32 *items, int cnt)
{
int unique = 1;
int i;
sort(items, cnt, sizeof(items[0]), cmp_ptr_to_u32, NULL);
for (i = 1; i < cnt; i++)
if (items[i] != items[unique - 1])
items[unique++] = items[i];
return unique;
}
/*
* sort_unique({map[start], ..., map[end]}) into off
*/
int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off)
{
u32 n = end - start + 1;
int err;
err = copy_insn_array(map, start, end, off);
if (err)
return err;
return sort_insn_array_uniq(off, n);
}
/*
* Copy all unique offsets from the map
*/
static struct bpf_iarray *jt_from_map(struct bpf_map *map)
{
struct bpf_iarray *jt;
int err;
int n;
jt = bpf_iarray_realloc(NULL, map->max_entries);
if (!jt)
return ERR_PTR(-ENOMEM);
n = bpf_copy_insn_array_uniq(map, 0, map->max_entries - 1, jt->items);
if (n < 0) {
err = n;
goto err_free;
}
if (n == 0) {
err = -EINVAL;
goto err_free;
}
jt->cnt = n;
return jt;
err_free:
kvfree(jt);
return ERR_PTR(err);
}
/*
* Find and collect all maps which fit in the subprog. Return the result as one
* combined jump table in jt->items (allocated with kvcalloc)
*/
static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env,
int subprog_start, int subprog_end)
{
struct bpf_iarray *jt = NULL;
struct bpf_map *map;
struct bpf_iarray *jt_cur;
int i;
for (i = 0; i < env->insn_array_map_cnt; i++) {
/*
* TODO (when needed): collect only jump tables, not static keys
* or maps for indirect calls
*/
map = env->insn_array_maps[i];
jt_cur = jt_from_map(map);
if (IS_ERR(jt_cur)) {
kvfree(jt);
return jt_cur;
}
/*
* This is enough to check one element. The full table is
* checked to fit inside the subprog later in create_jt()
*/
if (jt_cur->items[0] >= subprog_start && jt_cur->items[0] < subprog_end) {
u32 old_cnt = jt ? jt->cnt : 0;
jt = bpf_iarray_realloc(jt, old_cnt + jt_cur->cnt);
if (!jt) {
kvfree(jt_cur);
return ERR_PTR(-ENOMEM);
}
memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2);
}
kvfree(jt_cur);
}
if (!jt) {
verbose(env, "no jump tables found for subprog starting at %u\n", subprog_start);
return ERR_PTR(-EINVAL);
}
jt->cnt = sort_insn_array_uniq(jt->items, jt->cnt);
return jt;
}
static struct bpf_iarray *
create_jt(int t, struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprog;
int subprog_start, subprog_end;
struct bpf_iarray *jt;
int i;
subprog = bpf_find_containing_subprog(env, t);
subprog_start = subprog->start;
subprog_end = (subprog + 1)->start;
jt = jt_from_subprog(env, subprog_start, subprog_end);
if (IS_ERR(jt))
return jt;
/* Check that the every element of the jump table fits within the given subprogram */
for (i = 0; i < jt->cnt; i++) {
if (jt->items[i] < subprog_start || jt->items[i] >= subprog_end) {
verbose(env, "jump table for insn %d points outside of the subprog [%u,%u]\n",
t, subprog_start, subprog_end);
kvfree(jt);
return ERR_PTR(-EINVAL);
}
}
return jt;
}
/* "conditional jump with N edges" */
static int visit_gotox_insn(int t, struct bpf_verifier_env *env)
{
int *insn_stack = env->cfg.insn_stack;
int *insn_state = env->cfg.insn_state;
bool keep_exploring = false;
struct bpf_iarray *jt;
int i, w;
jt = env->insn_aux_data[t].jt;
if (!jt) {
jt = create_jt(t, env);
if (IS_ERR(jt))
return PTR_ERR(jt);
env->insn_aux_data[t].jt = jt;
}
mark_prune_point(env, t);
for (i = 0; i < jt->cnt; i++) {
w = jt->items[i];
if (w < 0 || w >= env->prog->len) {
verbose(env, "indirect jump out of range from insn %d to %d\n", t, w);
return -EINVAL;
}
mark_jmp_point(env, w);
/* EXPLORED || DISCOVERED */
if (insn_state[w])
continue;
if (env->cfg.cur_stack >= env->prog->len)
return -E2BIG;
insn_stack[env->cfg.cur_stack++] = w;
insn_state[w] |= DISCOVERED;
keep_exploring = true;
}
return keep_exploring ? KEEP_EXPLORING : DONE_EXPLORING;
}
/*
* Instructions that can abnormally return from a subprog (tail_call
* upon success, ld_{abs,ind} upon load failure) have a hidden exit
* that the verifier must account for.
*/
static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t)
{
struct bpf_subprog_info *subprog;
struct bpf_iarray *jt;
if (env->insn_aux_data[t].jt)
return 0;
jt = bpf_iarray_realloc(NULL, 2);
if (!jt)
return -ENOMEM;
subprog = bpf_find_containing_subprog(env, t);
jt->items[0] = t + 1;
jt->items[1] = subprog->exit_idx;
env->insn_aux_data[t].jt = jt;
return 0;
}
/* Visits the instruction at index t and returns one of the following:
* < 0 - an error occurred
* DONE_EXPLORING - the instruction was fully explored
* KEEP_EXPLORING - there is still work to be done before it is fully explored
*/
static int visit_insn(int t, struct bpf_verifier_env *env)
{
struct bpf_insn *insns = env->prog->insnsi, *insn = &insns[t];
int ret, off, insn_sz;
if (bpf_pseudo_func(insn))
return visit_func_call_insn(t, insns, env, true);
/* All non-branch instructions have a single fall-through edge. */
if (BPF_CLASS(insn->code) != BPF_JMP &&
BPF_CLASS(insn->code) != BPF_JMP32) {
if (BPF_CLASS(insn->code) == BPF_LD &&
(BPF_MODE(insn->code) == BPF_ABS ||
BPF_MODE(insn->code) == BPF_IND)) {
ret = visit_abnormal_return_insn(env, t);
if (ret)
return ret;
}
insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
return push_insn(t, t + insn_sz, FALLTHROUGH, env);
}
switch (BPF_OP(insn->code)) {
case BPF_EXIT:
return DONE_EXPLORING;
case BPF_CALL:
if (bpf_is_async_callback_calling_insn(insn))
/* Mark this call insn as a prune point to trigger
* is_state_visited() check before call itself is
* processed by __check_func_call(). Otherwise new
* async state will be pushed for further exploration.
*/
mark_prune_point(env, t);
/* For functions that invoke callbacks it is not known how many times
* callback would be called. Verifier models callback calling functions
* by repeatedly visiting callback bodies and returning to origin call
* instruction.
* In order to stop such iteration verifier needs to identify when a
* state identical some state from a previous iteration is reached.
* Check below forces creation of checkpoint before callback calling
* instruction to allow search for such identical states.
*/
if (bpf_is_sync_callback_calling_insn(insn)) {
mark_calls_callback(env, t);
mark_force_checkpoint(env, t);
mark_prune_point(env, t);
mark_jmp_point(env, t);
}
if (bpf_helper_call(insn)) {
const struct bpf_func_proto *fp;
ret = bpf_get_helper_proto(env, insn->imm, &fp);
/* If called in a non-sleepable context program will be
* rejected anyway, so we should end up with precise
* sleepable marks on subprogs, except for dead code
* elimination.
*/
if (ret == 0 && fp->might_sleep)
mark_subprog_might_sleep(env, t);
if (bpf_helper_changes_pkt_data(insn->imm))
mark_subprog_changes_pkt_data(env, t);
if (insn->imm == BPF_FUNC_tail_call) {
ret = visit_abnormal_return_insn(env, t);
if (ret)
return ret;
}
} else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
struct bpf_kfunc_call_arg_meta meta;
ret = bpf_fetch_kfunc_arg_meta(env, insn->imm, insn->off, &meta);
if (ret == 0 && bpf_is_iter_next_kfunc(&meta)) {
mark_prune_point(env, t);
/* Checking and saving state checkpoints at iter_next() call
* is crucial for fast convergence of open-coded iterator loop
* logic, so we need to force it. If we don't do that,
* is_state_visited() might skip saving a checkpoint, causing
* unnecessarily long sequence of not checkpointed
* instructions and jumps, leading to exhaustion of jump
* history buffer, and potentially other undesired outcomes.
* It is expected that with correct open-coded iterators
* convergence will happen quickly, so we don't run a risk of
* exhausting memory.
*/
mark_force_checkpoint(env, t);
}
/* Same as helpers, if called in a non-sleepable context
* program will be rejected anyway, so we should end up
* with precise sleepable marks on subprogs, except for
* dead code elimination.
*/
if (ret == 0 && bpf_is_kfunc_sleepable(&meta))
mark_subprog_might_sleep(env, t);
if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta))
mark_subprog_changes_pkt_data(env, t);
}
return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
case BPF_JA:
if (BPF_SRC(insn->code) == BPF_X)
return visit_gotox_insn(t, env);
if (BPF_CLASS(insn->code) == BPF_JMP)
off = insn->off;
else
off = insn->imm;
/* unconditional jump with single edge */
ret = push_insn(t, t + off + 1, FALLTHROUGH, env);
if (ret)
return ret;
mark_prune_point(env, t + off + 1);
mark_jmp_point(env, t + off + 1);
return ret;
default:
/* conditional jump with two edges */
mark_prune_point(env, t);
if (bpf_is_may_goto_insn(insn))
mark_force_checkpoint(env, t);
ret = push_insn(t, t + 1, FALLTHROUGH, env);
if (ret)
return ret;
return push_insn(t, t + insn->off + 1, BRANCH, env);
}
}
/* non-recursive depth-first-search to detect loops in BPF program
* loop == back-edge in directed graph
*/
int bpf_check_cfg(struct bpf_verifier_env *env)
{
int insn_cnt = env->prog->len;
int *insn_stack, *insn_state;
int ex_insn_beg, i, ret = 0;
insn_state = env->cfg.insn_state = kvzalloc_objs(int, insn_cnt,
GFP_KERNEL_ACCOUNT);
if (!insn_state)
return -ENOMEM;
insn_stack = env->cfg.insn_stack = kvzalloc_objs(int, insn_cnt,
GFP_KERNEL_ACCOUNT);
if (!insn_stack) {
kvfree(insn_state);
return -ENOMEM;
}
ex_insn_beg = env->exception_callback_subprog
? env->subprog_info[env->exception_callback_subprog].start
: 0;
insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
insn_stack[0] = 0; /* 0 is the first instruction */
env->cfg.cur_stack = 1;
walk_cfg:
while (env->cfg.cur_stack > 0) {
int t = insn_stack[env->cfg.cur_stack - 1];
ret = visit_insn(t, env);
switch (ret) {
case DONE_EXPLORING:
insn_state[t] = EXPLORED;
env->cfg.cur_stack--;
break;
case KEEP_EXPLORING:
break;
default:
if (ret > 0) {
verifier_bug(env, "visit_insn internal bug");
ret = -EFAULT;
}
goto err_free;
}
}
if (env->cfg.cur_stack < 0) {
verifier_bug(env, "pop stack internal bug");
ret = -EFAULT;
goto err_free;
}
if (ex_insn_beg && insn_state[ex_insn_beg] != EXPLORED) {
insn_state[ex_insn_beg] = DISCOVERED;
insn_stack[0] = ex_insn_beg;
env->cfg.cur_stack = 1;
goto walk_cfg;
}
for (i = 0; i < insn_cnt; i++) {
struct bpf_insn *insn = &env->prog->insnsi[i];
if (insn_state[i] != EXPLORED) {
verbose(env, "unreachable insn %d\n", i);
ret = -EINVAL;
goto err_free;
}
if (bpf_is_ldimm64(insn)) {
if (insn_state[i + 1] != 0) {
verbose(env, "jump into the middle of ldimm64 insn %d\n", i);
ret = -EINVAL;
goto err_free;
}
i++; /* skip second half of ldimm64 */
}
}
ret = 0; /* cfg looks good */
env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;
env->prog->aux->might_sleep = env->subprog_info[0].might_sleep;
err_free:
kvfree(insn_state);
kvfree(insn_stack);
env->cfg.insn_state = env->cfg.insn_stack = NULL;
return ret;
}
/*
* For each subprogram 'i' fill array env->cfg.insn_subprogram sub-range
* [env->subprog_info[i].postorder_start, env->subprog_info[i+1].postorder_start)
* with indices of 'i' instructions in postorder.
*/
int bpf_compute_postorder(struct bpf_verifier_env *env)
{
u32 cur_postorder, i, top, stack_sz, s;
int *stack = NULL, *postorder = NULL, *state = NULL;
struct bpf_iarray *succ;
postorder = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
state = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
stack = kvzalloc_objs(int, env->prog->len, GFP_KERNEL_ACCOUNT);
if (!postorder || !state || !stack) {
kvfree(postorder);
kvfree(state);
kvfree(stack);
return -ENOMEM;
}
cur_postorder = 0;
for (i = 0; i < env->subprog_cnt; i++) {
env->subprog_info[i].postorder_start = cur_postorder;
stack[0] = env->subprog_info[i].start;
stack_sz = 1;
do {
top = stack[stack_sz - 1];
state[top] |= DISCOVERED;
if (state[top] & EXPLORED) {
postorder[cur_postorder++] = top;
stack_sz--;
continue;
}
succ = bpf_insn_successors(env, top);
for (s = 0; s < succ->cnt; ++s) {
if (!state[succ->items[s]]) {
stack[stack_sz++] = succ->items[s];
state[succ->items[s]] |= DISCOVERED;
}
}
state[top] |= EXPLORED;
} while (stack_sz);
}
env->subprog_info[i].postorder_start = cur_postorder;
env->cfg.insn_postorder = postorder;
env->cfg.cur_postorder = cur_postorder;
kvfree(stack);
kvfree(state);
return 0;
}
/*
* Compute strongly connected components (SCCs) on the CFG.
* Assign an SCC number to each instruction, recorded in env->insn_aux[*].scc.
* If instruction is a sole member of its SCC and there are no self edges,
* assign it SCC number of zero.
* Uses a non-recursive adaptation of Tarjan's algorithm for SCC computation.
*/
int bpf_compute_scc(struct bpf_verifier_env *env)
{
const u32 NOT_ON_STACK = U32_MAX;
struct bpf_insn_aux_data *aux = env->insn_aux_data;
const u32 insn_cnt = env->prog->len;
int stack_sz, dfs_sz, err = 0;
u32 *stack, *pre, *low, *dfs;
u32 i, j, t, w;
u32 next_preorder_num;
u32 next_scc_id;
bool assign_scc;
struct bpf_iarray *succ;
next_preorder_num = 1;
next_scc_id = 1;
/*
* - 'stack' accumulates vertices in DFS order, see invariant comment below;
* - 'pre[t] == p' => preorder number of vertex 't' is 'p';
* - 'low[t] == n' => smallest preorder number of the vertex reachable from 't' is 'n';
* - 'dfs' DFS traversal stack, used to emulate explicit recursion.
*/
stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
pre = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
low = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL_ACCOUNT);
dfs = kvcalloc(insn_cnt, sizeof(*dfs), GFP_KERNEL_ACCOUNT);
if (!stack || !pre || !low || !dfs) {
err = -ENOMEM;
goto exit;
}
/*
* References:
* [1] R. Tarjan "Depth-First Search and Linear Graph Algorithms"
* [2] D. J. Pearce "A Space-Efficient Algorithm for Finding Strongly Connected Components"
*
* The algorithm maintains the following invariant:
* - suppose there is a path 'u' ~> 'v', such that 'pre[v] < pre[u]';
* - then, vertex 'u' remains on stack while vertex 'v' is on stack.
*
* Consequently:
* - If 'low[v] < pre[v]', there is a path from 'v' to some vertex 'u',
* such that 'pre[u] == low[v]'; vertex 'u' is currently on the stack,
* and thus there is an SCC (loop) containing both 'u' and 'v'.
* - If 'low[v] == pre[v]', loops containing 'v' have been explored,
* and 'v' can be considered the root of some SCC.
*
* Here is a pseudo-code for an explicitly recursive version of the algorithm:
*
* NOT_ON_STACK = insn_cnt + 1
* pre = [0] * insn_cnt
* low = [0] * insn_cnt
* scc = [0] * insn_cnt
* stack = []
*
* next_preorder_num = 1
* next_scc_id = 1
*
* def recur(w):
* nonlocal next_preorder_num
* nonlocal next_scc_id
*
* pre[w] = next_preorder_num
* low[w] = next_preorder_num
* next_preorder_num += 1
* stack.append(w)
* for s in successors(w):
* # Note: for classic algorithm the block below should look as:
* #
* # if pre[s] == 0:
* # recur(s)
* # low[w] = min(low[w], low[s])
* # elif low[s] != NOT_ON_STACK:
* # low[w] = min(low[w], pre[s])
* #
* # But replacing both 'min' instructions with 'low[w] = min(low[w], low[s])'
* # does not break the invariant and makes iterative version of the algorithm
* # simpler. See 'Algorithm #3' from [2].
*
* # 's' not yet visited
* if pre[s] == 0:
* recur(s)
* # if 's' is on stack, pick lowest reachable preorder number from it;
* # if 's' is not on stack 'low[s] == NOT_ON_STACK > low[w]',
* # so 'min' would be a noop.
* low[w] = min(low[w], low[s])
*
* if low[w] == pre[w]:
* # 'w' is the root of an SCC, pop all vertices
* # below 'w' on stack and assign same SCC to them.
* while True:
* t = stack.pop()
* low[t] = NOT_ON_STACK
* scc[t] = next_scc_id
* if t == w:
* break
* next_scc_id += 1
*
* for i in range(0, insn_cnt):
* if pre[i] == 0:
* recur(i)
*
* Below implementation replaces explicit recursion with array 'dfs'.
*/
for (i = 0; i < insn_cnt; i++) {
if (pre[i])
continue;
stack_sz = 0;
dfs_sz = 1;
dfs[0] = i;
dfs_continue:
while (dfs_sz) {
w = dfs[dfs_sz - 1];
if (pre[w] == 0) {
low[w] = next_preorder_num;
pre[w] = next_preorder_num;
next_preorder_num++;
stack[stack_sz++] = w;
}
/* Visit 'w' successors */
succ = bpf_insn_successors(env, w);
for (j = 0; j < succ->cnt; ++j) {
if (pre[succ->items[j]]) {
low[w] = min(low[w], low[succ->items[j]]);
} else {
dfs[dfs_sz++] = succ->items[j];
goto dfs_continue;
}
}
/*
* Preserve the invariant: if some vertex above in the stack
* is reachable from 'w', keep 'w' on the stack.
*/
if (low[w] < pre[w]) {
dfs_sz--;
goto dfs_continue;
}
/*
* Assign SCC number only if component has two or more elements,
* or if component has a self reference, or if instruction is a
* callback calling function (implicit loop).
*/
assign_scc = stack[stack_sz - 1] != w; /* two or more elements? */
for (j = 0; j < succ->cnt; ++j) { /* self reference? */
if (succ->items[j] == w) {
assign_scc = true;
break;
}
}
if (bpf_calls_callback(env, w)) /* implicit loop? */
assign_scc = true;
/* Pop component elements from stack */
do {
t = stack[--stack_sz];
low[t] = NOT_ON_STACK;
if (assign_scc)
aux[t].scc = next_scc_id;
} while (t != w);
if (assign_scc)
next_scc_id++;
dfs_sz--;
}
}
env->scc_info = kvzalloc_objs(*env->scc_info, next_scc_id,
GFP_KERNEL_ACCOUNT);
if (!env->scc_info) {
err = -ENOMEM;
goto exit;
}
env->scc_cnt = next_scc_id;
exit:
kvfree(stack);
kvfree(pre);
kvfree(low);
kvfree(dfs);
return err;
}

463
kernel/bpf/check_btf.c Normal file
View File

@ -0,0 +1,463 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
#include <linux/bpf.h>
#include <linux/bpf_verifier.h>
#include <linux/filter.h>
#include <linux/btf.h>
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
static int check_abnormal_return(struct bpf_verifier_env *env)
{
int i;
for (i = 1; i < env->subprog_cnt; i++) {
if (env->subprog_info[i].has_ld_abs) {
verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
return -EINVAL;
}
if (env->subprog_info[i].has_tail_call) {
verbose(env, "tail_call is not allowed in subprogs without BTF\n");
return -EINVAL;
}
}
return 0;
}
/* The minimum supported BTF func info size */
#define MIN_BPF_FUNCINFO_SIZE 8
#define MAX_FUNCINFO_REC_SIZE 252
static int check_btf_func_early(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
u32 krec_size = sizeof(struct bpf_func_info);
const struct btf_type *type, *func_proto;
u32 i, nfuncs, urec_size, min_size;
struct bpf_func_info *krecord;
struct bpf_prog *prog;
const struct btf *btf;
u32 prev_offset = 0;
bpfptr_t urecord;
int ret = -ENOMEM;
nfuncs = attr->func_info_cnt;
if (!nfuncs) {
if (check_abnormal_return(env))
return -EINVAL;
return 0;
}
urec_size = attr->func_info_rec_size;
if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
urec_size > MAX_FUNCINFO_REC_SIZE ||
urec_size % sizeof(u32)) {
verbose(env, "invalid func info rec size %u\n", urec_size);
return -EINVAL;
}
prog = env->prog;
btf = prog->aux->btf;
urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
min_size = min_t(u32, krec_size, urec_size);
krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!krecord)
return -ENOMEM;
for (i = 0; i < nfuncs; i++) {
ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
if (ret) {
if (ret == -E2BIG) {
verbose(env, "nonzero tailing record in func info");
/* set the size kernel expects so loader can zero
* out the rest of the record.
*/
if (copy_to_bpfptr_offset(uattr,
offsetof(union bpf_attr, func_info_rec_size),
&min_size, sizeof(min_size)))
ret = -EFAULT;
}
goto err_free;
}
if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
ret = -EFAULT;
goto err_free;
}
/* check insn_off */
ret = -EINVAL;
if (i == 0) {
if (krecord[i].insn_off) {
verbose(env,
"nonzero insn_off %u for the first func info record",
krecord[i].insn_off);
goto err_free;
}
} else if (krecord[i].insn_off <= prev_offset) {
verbose(env,
"same or smaller insn offset (%u) than previous func info record (%u)",
krecord[i].insn_off, prev_offset);
goto err_free;
}
/* check type_id */
type = btf_type_by_id(btf, krecord[i].type_id);
if (!type || !btf_type_is_func(type)) {
verbose(env, "invalid type id %d in func info",
krecord[i].type_id);
goto err_free;
}
func_proto = btf_type_by_id(btf, type->type);
if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
/* btf_func_check() already verified it during BTF load */
goto err_free;
prev_offset = krecord[i].insn_off;
bpfptr_add(&urecord, urec_size);
}
prog->aux->func_info = krecord;
prog->aux->func_info_cnt = nfuncs;
return 0;
err_free:
kvfree(krecord);
return ret;
}
static int check_btf_func(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
const struct btf_type *type, *func_proto, *ret_type;
u32 i, nfuncs, urec_size;
struct bpf_func_info *krecord;
struct bpf_func_info_aux *info_aux = NULL;
struct bpf_prog *prog;
const struct btf *btf;
bpfptr_t urecord;
bool scalar_return;
int ret = -ENOMEM;
nfuncs = attr->func_info_cnt;
if (!nfuncs) {
if (check_abnormal_return(env))
return -EINVAL;
return 0;
}
if (nfuncs != env->subprog_cnt) {
verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
return -EINVAL;
}
urec_size = attr->func_info_rec_size;
prog = env->prog;
btf = prog->aux->btf;
urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
krecord = prog->aux->func_info;
info_aux = kzalloc_objs(*info_aux, nfuncs,
GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!info_aux)
return -ENOMEM;
for (i = 0; i < nfuncs; i++) {
/* check insn_off */
ret = -EINVAL;
if (env->subprog_info[i].start != krecord[i].insn_off) {
verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
goto err_free;
}
/* Already checked type_id */
type = btf_type_by_id(btf, krecord[i].type_id);
info_aux[i].linkage = BTF_INFO_VLEN(type->info);
/* Already checked func_proto */
func_proto = btf_type_by_id(btf, type->type);
ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
scalar_return =
btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);
if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
goto err_free;
}
if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
goto err_free;
}
env->subprog_info[i].name = btf_name_by_offset(btf, type->name_off);
bpfptr_add(&urecord, urec_size);
}
prog->aux->func_info_aux = info_aux;
return 0;
err_free:
kfree(info_aux);
return ret;
}
#define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col)
#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
static int check_btf_line(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
struct bpf_subprog_info *sub;
struct bpf_line_info *linfo;
struct bpf_prog *prog;
const struct btf *btf;
bpfptr_t ulinfo;
int err;
nr_linfo = attr->line_info_cnt;
if (!nr_linfo)
return 0;
if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))
return -EINVAL;
rec_size = attr->line_info_rec_size;
if (rec_size < MIN_BPF_LINEINFO_SIZE ||
rec_size > MAX_LINEINFO_REC_SIZE ||
rec_size & (sizeof(u32) - 1))
return -EINVAL;
/* Need to zero it in case the userspace may
* pass in a smaller bpf_line_info object.
*/
linfo = kvzalloc_objs(struct bpf_line_info, nr_linfo,
GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
if (!linfo)
return -ENOMEM;
prog = env->prog;
btf = prog->aux->btf;
s = 0;
sub = env->subprog_info;
ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
expected_size = sizeof(struct bpf_line_info);
ncopy = min_t(u32, expected_size, rec_size);
for (i = 0; i < nr_linfo; i++) {
err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
if (err) {
if (err == -E2BIG) {
verbose(env, "nonzero tailing record in line_info");
if (copy_to_bpfptr_offset(uattr,
offsetof(union bpf_attr, line_info_rec_size),
&expected_size, sizeof(expected_size)))
err = -EFAULT;
}
goto err_free;
}
if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
err = -EFAULT;
goto err_free;
}
/*
* Check insn_off to ensure
* 1) strictly increasing AND
* 2) bounded by prog->len
*
* The linfo[0].insn_off == 0 check logically falls into
* the later "missing bpf_line_info for func..." case
* because the first linfo[0].insn_off must be the
* first sub also and the first sub must have
* subprog_info[0].start == 0.
*/
if ((i && linfo[i].insn_off <= prev_offset) ||
linfo[i].insn_off >= prog->len) {
verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
i, linfo[i].insn_off, prev_offset,
prog->len);
err = -EINVAL;
goto err_free;
}
if (!prog->insnsi[linfo[i].insn_off].code) {
verbose(env,
"Invalid insn code at line_info[%u].insn_off\n",
i);
err = -EINVAL;
goto err_free;
}
if (!btf_name_by_offset(btf, linfo[i].line_off) ||
!btf_name_by_offset(btf, linfo[i].file_name_off)) {
verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
err = -EINVAL;
goto err_free;
}
if (s != env->subprog_cnt) {
if (linfo[i].insn_off == sub[s].start) {
sub[s].linfo_idx = i;
s++;
} else if (sub[s].start < linfo[i].insn_off) {
verbose(env, "missing bpf_line_info for func#%u\n", s);
err = -EINVAL;
goto err_free;
}
}
prev_offset = linfo[i].insn_off;
bpfptr_add(&ulinfo, rec_size);
}
if (s != env->subprog_cnt) {
verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
env->subprog_cnt - s, s);
err = -EINVAL;
goto err_free;
}
prog->aux->linfo = linfo;
prog->aux->nr_linfo = nr_linfo;
return 0;
err_free:
kvfree(linfo);
return err;
}
#define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo)
#define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE
static int check_core_relo(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
u32 i, nr_core_relo, ncopy, expected_size, rec_size;
struct bpf_core_relo core_relo = {};
struct bpf_prog *prog = env->prog;
const struct btf *btf = prog->aux->btf;
struct bpf_core_ctx ctx = {
.log = &env->log,
.btf = btf,
};
bpfptr_t u_core_relo;
int err;
nr_core_relo = attr->core_relo_cnt;
if (!nr_core_relo)
return 0;
if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))
return -EINVAL;
rec_size = attr->core_relo_rec_size;
if (rec_size < MIN_CORE_RELO_SIZE ||
rec_size > MAX_CORE_RELO_SIZE ||
rec_size % sizeof(u32))
return -EINVAL;
u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);
expected_size = sizeof(struct bpf_core_relo);
ncopy = min_t(u32, expected_size, rec_size);
/* Unlike func_info and line_info, copy and apply each CO-RE
* relocation record one at a time.
*/
for (i = 0; i < nr_core_relo; i++) {
/* future proofing when sizeof(bpf_core_relo) changes */
err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);
if (err) {
if (err == -E2BIG) {
verbose(env, "nonzero tailing record in core_relo");
if (copy_to_bpfptr_offset(uattr,
offsetof(union bpf_attr, core_relo_rec_size),
&expected_size, sizeof(expected_size)))
err = -EFAULT;
}
break;
}
if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {
err = -EFAULT;
break;
}
if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {
verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",
i, core_relo.insn_off, prog->len);
err = -EINVAL;
break;
}
err = bpf_core_apply(&ctx, &core_relo, i,
&prog->insnsi[core_relo.insn_off / 8]);
if (err)
break;
bpfptr_add(&u_core_relo, rec_size);
}
return err;
}
int bpf_check_btf_info_early(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
struct btf *btf;
int err;
if (!attr->func_info_cnt && !attr->line_info_cnt) {
if (check_abnormal_return(env))
return -EINVAL;
return 0;
}
btf = btf_get_by_fd(attr->prog_btf_fd);
if (IS_ERR(btf))
return PTR_ERR(btf);
if (btf_is_kernel(btf)) {
btf_put(btf);
return -EACCES;
}
env->prog->aux->btf = btf;
err = check_btf_func_early(env, attr, uattr);
if (err)
return err;
return 0;
}
int bpf_check_btf_info(struct bpf_verifier_env *env,
const union bpf_attr *attr,
bpfptr_t uattr)
{
int err;
if (!attr->func_info_cnt && !attr->line_info_cnt) {
if (check_abnormal_return(env))
return -EINVAL;
return 0;
}
err = check_btf_func(env, attr, uattr);
if (err)
return err;
err = check_btf_line(env, attr, uattr);
if (err)
return err;
err = check_core_relo(env, attr, uattr);
if (err)
return err;
return 0;
}

2457
kernel/bpf/fixups.c Normal file
View File

File diff suppressed because it is too large Load Diff

247
kernel/bpf/liveness.c
View File

@ -1953,3 +1953,250 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env)
kvfree(info);
return err;
}
/* Each field is a register bitmask */
struct insn_live_regs {
u16 use; /* registers read by instruction */
u16 def; /* registers written by instruction */
u16 in; /* registers that may be alive before instruction */
u16 out; /* registers that may be alive after instruction */
};
/* Bitmask with 1s for all caller saved registers */
#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
/* Compute info->{use,def} fields for the instruction */
static void compute_insn_live_regs(struct bpf_verifier_env *env,
struct bpf_insn *insn,
struct insn_live_regs *info)
{
struct bpf_call_summary cs;
u8 class = BPF_CLASS(insn->code);
u8 code = BPF_OP(insn->code);
u8 mode = BPF_MODE(insn->code);
u16 src = BIT(insn->src_reg);
u16 dst = BIT(insn->dst_reg);
u16 r0 = BIT(0);
u16 def = 0;
u16 use = 0xffff;
switch (class) {
case BPF_LD:
switch (mode) {
case BPF_IMM:
if (BPF_SIZE(insn->code) == BPF_DW) {
def = dst;
use = 0;
}
break;
case BPF_LD | BPF_ABS:
case BPF_LD | BPF_IND:
/* stick with defaults */
break;
}
break;
case BPF_LDX:
switch (mode) {
case BPF_MEM:
case BPF_MEMSX:
def = dst;
use = src;
break;
}
break;
case BPF_ST:
switch (mode) {
case BPF_MEM:
def = 0;
use = dst;
break;
}
break;
case BPF_STX:
switch (mode) {
case BPF_MEM:
def = 0;
use = dst | src;
break;
case BPF_ATOMIC:
switch (insn->imm) {
case BPF_CMPXCHG:
use = r0 | dst | src;
def = r0;
break;
case BPF_LOAD_ACQ:
def = dst;
use = src;
break;
case BPF_STORE_REL:
def = 0;
use = dst | src;
break;
default:
use = dst | src;
if (insn->imm & BPF_FETCH)
def = src;
else
def = 0;
}
break;
}
break;
case BPF_ALU:
case BPF_ALU64:
switch (code) {
case BPF_END:
use = dst;
def = dst;
break;
case BPF_MOV:
def = dst;
if (BPF_SRC(insn->code) == BPF_K)
use = 0;
else
use = src;
break;
default:
def = dst;
if (BPF_SRC(insn->code) == BPF_K)
use = dst;
else
use = dst | src;
}
break;
case BPF_JMP:
case BPF_JMP32:
switch (code) {
case BPF_JA:
def = 0;
if (BPF_SRC(insn->code) == BPF_X)
use = dst;
else
use = 0;
break;
case BPF_JCOND:
def = 0;
use = 0;
break;
case BPF_EXIT:
def = 0;
use = r0;
break;
case BPF_CALL:
def = ALL_CALLER_SAVED_REGS;
use = def & ~BIT(BPF_REG_0);
if (bpf_get_call_summary(env, insn, &cs))
use = GENMASK(cs.num_params, 1);
break;
default:
def = 0;
if (BPF_SRC(insn->code) == BPF_K)
use = dst;
else
use = dst | src;
}
break;
}
info->def = def;
info->use = use;
}
/* Compute may-live registers after each instruction in the program.
* The register is live after the instruction I if it is read by some
* instruction S following I during program execution and is not
* overwritten between I and S.
*
* Store result in env->insn_aux_data[i].live_regs.
*/
int bpf_compute_live_registers(struct bpf_verifier_env *env)
{
struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
struct bpf_insn *insns = env->prog->insnsi;
struct insn_live_regs *state;
int insn_cnt = env->prog->len;
int err = 0, i, j;
bool changed;
/* Use the following algorithm:
* - define the following:
* - I.use : a set of all registers read by instruction I;
* - I.def : a set of all registers written by instruction I;
* - I.in : a set of all registers that may be alive before I execution;
* - I.out : a set of all registers that may be alive after I execution;
* - insn_successors(I): a set of instructions S that might immediately
* follow I for some program execution;
* - associate separate empty sets 'I.in' and 'I.out' with each instruction;
* - visit each instruction in a postorder and update
* state[i].in, state[i].out as follows:
*
* state[i].out = U [state[s].in for S in insn_successors(i)]
* state[i].in = (state[i].out / state[i].def) U state[i].use
*
* (where U stands for set union, / stands for set difference)
* - repeat the computation while {in,out} fields changes for
* any instruction.
*/
state = kvzalloc_objs(*state, insn_cnt, GFP_KERNEL_ACCOUNT);
if (!state) {
err = -ENOMEM;
goto out;
}
for (i = 0; i < insn_cnt; ++i)
compute_insn_live_regs(env, &insns[i], &state[i]);
/* Forward pass: resolve stack access through FP-derived pointers */
err = bpf_compute_subprog_arg_access(env);
if (err)
goto out;
changed = true;
while (changed) {
changed = false;
for (i = 0; i < env->cfg.cur_postorder; ++i) {
int insn_idx = env->cfg.insn_postorder[i];
struct insn_live_regs *live = &state[insn_idx];
struct bpf_iarray *succ;
u16 new_out = 0;
u16 new_in = 0;
succ = bpf_insn_successors(env, insn_idx);
for (int s = 0; s < succ->cnt; ++s)
new_out |= state[succ->items[s]].in;
new_in = (new_out & ~live->def) | live->use;
if (new_out != live->out || new_in != live->in) {
live->in = new_in;
live->out = new_out;
changed = true;
}
}
}
for (i = 0; i < insn_cnt; ++i)
insn_aux[i].live_regs_before = state[i].in;
if (env->log.level & BPF_LOG_LEVEL2) {
verbose(env, "Live regs before insn:\n");
for (i = 0; i < insn_cnt; ++i) {
if (env->insn_aux_data[i].scc)
verbose(env, "%3d ", env->insn_aux_data[i].scc);
else
verbose(env, " ");
verbose(env, "%3d: ", i);
for (j = BPF_REG_0; j < BPF_REG_10; ++j)
if (insn_aux[i].live_regs_before & BIT(j))
verbose(env, "%d", j);
else
verbose(env, ".");
verbose(env, " ");
bpf_verbose_insn(env, &insns[i]);
if (bpf_is_ldimm64(&insns[i]))
i++;
}
}
out:
kvfree(state);
return err;
}

1563
kernel/bpf/states.c Normal file
View File

File diff suppressed because it is too large Load Diff

7435
kernel/bpf/verifier.c
View File

File diff suppressed because it is too large Load Diff