github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-28 18:21:24 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: make bpf_insn_successors to return a pointer

Browse Source 
The bpf_insn_successors() function is used to return successors
to a BPF instruction. So far, an instruction could have 0, 1 or 2
successors. Prepare the verifier code to introduction of instructions
with more than 2 successors (namely, indirect jumps).

To do this, introduce a new struct, struct bpf_iarray, containing
an array of bpf instruction indexes and make bpf_insn_successors
to return a pointer of that type. The storage for all instructions
is allocated in the env->succ, which holds an array of size 2,
to be used for all instructions.

Signed-off-by: Anton Protopopov <a.s.protopopov@gmail.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251019202145.3944697-10-a.s.protopopov@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Anton Protopopov 2025-10-19 20:21:37 +00:00 committed by Alexei Starovoitov
parent 44481e4925
commit 2f69c56854
3 changed files with 75 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/linux/bpf_verifier.h
View File

@ -509,6 +509,15 @@ struct bpf_map_ptr_state {
#define BPF_ALU_SANITIZE (BPF_ALU_SANITIZE_SRC | \ #define BPF_ALU_SANITIZE (BPF_ALU_SANITIZE_SRC | \
BPF_ALU_SANITIZE_DST) BPF_ALU_SANITIZE_DST)
/*
* An array of BPF instructions.
* Primary usage: return value of bpf_insn_successors.
*/
struct bpf_iarray {
int cnt;
u32 items[];
};
struct bpf_insn_aux_data { struct bpf_insn_aux_data {
union { union {
enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ enum bpf_reg_type ptr_type; /* pointer type for load/store insns */
@ -828,6 +837,7 @@ struct bpf_verifier_env {
/* array of pointers to bpf_scc_info indexed by SCC id */ /* array of pointers to bpf_scc_info indexed by SCC id */
struct bpf_scc_info **scc_info; struct bpf_scc_info **scc_info;
u32 scc_cnt; u32 scc_cnt;
struct bpf_iarray *succ;
}; };
static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog) static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
@ -1050,7 +1060,7 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st
struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off); struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);
int bpf_jmp_offset(struct bpf_insn *insn); int bpf_jmp_offset(struct bpf_insn *insn);
int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2]); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx); bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);

36
kernel/bpf/liveness.c
View File

@ -34,7 +34,7 @@
* - read and write marks propagation. * - read and write marks propagation.
* - The propagation phase is a textbook live variable data flow analysis: * - The propagation phase is a textbook live variable data flow analysis:
* *
* state[cc, i].live_after = U [state[cc, s].live_before for s in insn_successors(i)] * state[cc, i].live_after = U [state[cc, s].live_before for s in bpf_insn_successors(i)]
* state[cc, i].live_before = * state[cc, i].live_before =
* (state[cc, i].live_after / state[cc, i].must_write) U state[i].may_read * (state[cc, i].live_after / state[cc, i].must_write) U state[i].may_read
* *
@ -54,7 +54,7 @@
* The equation for "must_write_acc" propagation looks as follows: * The equation for "must_write_acc" propagation looks as follows:
* *
* state[cc, i].must_write_acc = * state[cc, i].must_write_acc =
* ∩ [state[cc, s].must_write_acc for s in insn_successors(i)] * ∩ [state[cc, s].must_write_acc for s in bpf_insn_successors(i)]
* U state[cc, i].must_write * U state[cc, i].must_write
* *
* (An intersection of all "must_write_acc" for instruction successors * (An intersection of all "must_write_acc" for instruction successors
@ -447,7 +447,12 @@ int bpf_jmp_offset(struct bpf_insn *insn)
__diag_push(); __diag_push();
__diag_ignore_all("-Woverride-init", "Allow field initialization overrides for opcode_info_tbl"); __diag_ignore_all("-Woverride-init", "Allow field initialization overrides for opcode_info_tbl");
inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2]) /*
* Returns an array of instructions succ, with succ->items[0], ...,
* succ->items[n-1] with successor instructions, where n=succ->cnt
*/
inline struct bpf_iarray *
bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)
{ {
static const struct opcode_info { static const struct opcode_info {
bool can_jump; bool can_jump;
@ -474,19 +479,25 @@ inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])
_J(BPF_JSET, {.can_jump = true, .can_fallthrough = true}), _J(BPF_JSET, {.can_jump = true, .can_fallthrough = true}),
#undef _J #undef _J
}; };
struct bpf_prog *prog = env->prog;
struct bpf_insn *insn = &prog->insnsi[idx]; struct bpf_insn *insn = &prog->insnsi[idx];
const struct opcode_info *opcode_info; const struct opcode_info *opcode_info;
int i = 0, insn_sz; struct bpf_iarray *succ;
int insn_sz;
/* pre-allocated array of size up to 2; reset cnt, as it may have been used already */
succ = env->succ;
succ->cnt = 0;
opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)]; opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)];
insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
if (opcode_info->can_fallthrough) if (opcode_info->can_fallthrough)
succ[i++] = idx + insn_sz; succ->items[succ->cnt++] = idx + insn_sz;
if (opcode_info->can_jump) if (opcode_info->can_jump)
succ[i++] = idx + bpf_jmp_offset(insn) + 1; succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1;
return i; return succ;
} }
__diag_pop(); __diag_pop();
@ -548,11 +559,12 @@ static inline bool update_insn(struct bpf_verifier_env *env,
struct bpf_insn_aux_data *aux = env->insn_aux_data; struct bpf_insn_aux_data *aux = env->insn_aux_data;
u64 new_before, new_after, must_write_acc; u64 new_before, new_after, must_write_acc;
struct per_frame_masks *insn, *succ_insn; struct per_frame_masks *insn, *succ_insn;
u32 succ_num, s, succ[2]; struct bpf_iarray *succ;
u32 s;
bool changed; bool changed;
succ_num = bpf_insn_successors(env->prog, insn_idx, succ); succ = bpf_insn_successors(env, insn_idx);
if (unlikely(succ_num == 0)) if (succ->cnt == 0)
return false; return false;
changed = false; changed = false;
@ -564,8 +576,8 @@ static inline bool update_insn(struct bpf_verifier_env *env,
* of successors plus all "must_write" slots of instruction itself. * of successors plus all "must_write" slots of instruction itself.
*/ */
must_write_acc = U64_MAX; must_write_acc = U64_MAX;
for (s = 0; s < succ_num; ++s) { for (s = 0; s < succ->cnt; ++s) {
succ_insn = get_frame_masks(instance, frame, succ[s]); succ_insn = get_frame_masks(instance, frame, succ->items[s]);
new_after |= succ_insn->live_before; new_after |= succ_insn->live_before;
must_write_acc &= succ_insn->must_write_acc; must_write_acc &= succ_insn->must_write_acc;
} }

60
kernel/bpf/verifier.c
View File

@ -17805,6 +17805,22 @@ static int mark_fastcall_patterns(struct bpf_verifier_env *env)
return 0; return 0;
} }
static struct bpf_iarray *iarray_realloc(struct bpf_iarray *old, size_t n_elem)
{
size_t new_size = sizeof(struct bpf_iarray) + n_elem * sizeof(old->items[0]);
struct bpf_iarray *new;
new = kvrealloc(old, new_size, GFP_KERNEL_ACCOUNT);
if (!new) {
/* this is what callers always want, so simplify the call site */
kvfree(old);
return NULL;
}
new->cnt = n_elem;
return new;
}
/* Visits the instruction at index t and returns one of the following: /* Visits the instruction at index t and returns one of the following:
* < 0 - an error occurred * < 0 - an error occurred
* DONE_EXPLORING - the instruction was fully explored * DONE_EXPLORING - the instruction was fully explored
@ -18025,8 +18041,9 @@ static int check_cfg(struct bpf_verifier_env *env)
*/ */
static int compute_postorder(struct bpf_verifier_env *env) static int compute_postorder(struct bpf_verifier_env *env)
{ {
u32 cur_postorder, i, top, stack_sz, s, succ_cnt, succ[2]; u32 cur_postorder, i, top, stack_sz, s;
int *stack = NULL, *postorder = NULL, *state = NULL; int *stack = NULL, *postorder = NULL, *state = NULL;
struct bpf_iarray *succ;
postorder = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT); postorder = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
state = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT); state = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
@ -18050,11 +18067,11 @@ static int compute_postorder(struct bpf_verifier_env *env)
stack_sz--; stack_sz--;
continue; continue;
} }
succ_cnt = bpf_insn_successors(env->prog, top, succ); succ = bpf_insn_successors(env, top);
for (s = 0; s < succ_cnt; ++s) { for (s = 0; s < succ->cnt; ++s) {
if (!state[succ[s]]) { if (!state[succ->items[s]]) {
stack[stack_sz++] = succ[s]; stack[stack_sz++] = succ->items[s];
state[succ[s]] |= DISCOVERED; state[succ->items[s]] |= DISCOVERED;
} }
} }
state[top] |= EXPLORED; state[top] |= EXPLORED;
@ -24313,14 +24330,13 @@ static int compute_live_registers(struct bpf_verifier_env *env)
for (i = 0; i < env->cfg.cur_postorder; ++i) { for (i = 0; i < env->cfg.cur_postorder; ++i) {
int insn_idx = env->cfg.insn_postorder[i]; int insn_idx = env->cfg.insn_postorder[i];
struct insn_live_regs *live = &state[insn_idx]; struct insn_live_regs *live = &state[insn_idx];
int succ_num; struct bpf_iarray *succ;
u32 succ[2];
u16 new_out = 0; u16 new_out = 0;
u16 new_in = 0; u16 new_in = 0;
succ_num = bpf_insn_successors(env->prog, insn_idx, succ); succ = bpf_insn_successors(env, insn_idx);
for (int s = 0; s < succ_num; ++s) for (int s = 0; s < succ->cnt; ++s)
new_out |= state[succ[s]].in; new_out |= state[succ->items[s]].in;
new_in = (new_out & ~live->def) | live->use; new_in = (new_out & ~live->def) | live->use;
if (new_out != live->out || new_in != live->in) { if (new_out != live->out || new_in != live->in) {
live->in = new_in; live->in = new_in;
@ -24373,11 +24389,11 @@ static int compute_scc(struct bpf_verifier_env *env)
const u32 insn_cnt = env->prog->len; const u32 insn_cnt = env->prog->len;
int stack_sz, dfs_sz, err = 0; int stack_sz, dfs_sz, err = 0;
u32 *stack, *pre, *low, *dfs; u32 *stack, *pre, *low, *dfs;
u32 succ_cnt, i, j, t, w; u32 i, j, t, w;
u32 next_preorder_num; u32 next_preorder_num;
u32 next_scc_id; u32 next_scc_id;
bool assign_scc; bool assign_scc;
u32 succ[2]; struct bpf_iarray *succ;
next_preorder_num = 1; next_preorder_num = 1;
next_scc_id = 1; next_scc_id = 1;
@ -24484,12 +24500,12 @@ static int compute_scc(struct bpf_verifier_env *env)
stack[stack_sz++] = w; stack[stack_sz++] = w;
} }
/* Visit 'w' successors */ /* Visit 'w' successors */
succ_cnt = bpf_insn_successors(env->prog, w, succ); succ = bpf_insn_successors(env, w);
for (j = 0; j < succ_cnt; ++j) { for (j = 0; j < succ->cnt; ++j) {
if (pre[succ[j]]) { if (pre[succ->items[j]]) {
low[w] = min(low[w], low[succ[j]]); low[w] = min(low[w], low[succ->items[j]]);
} else { } else {
dfs[dfs_sz++] = succ[j]; dfs[dfs_sz++] = succ->items[j];
goto dfs_continue; goto dfs_continue;
} }
} }
@ -24506,8 +24522,8 @@ static int compute_scc(struct bpf_verifier_env *env)
* or if component has a self reference. * or if component has a self reference.
*/ */
assign_scc = stack[stack_sz - 1] != w; assign_scc = stack[stack_sz - 1] != w;
for (j = 0; j < succ_cnt; ++j) { for (j = 0; j < succ->cnt; ++j) {
if (succ[j] == w) { if (succ->items[j] == w) {
assign_scc = true; assign_scc = true;
break; break;
} }
@ -24569,6 +24585,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
goto err_free_env; goto err_free_env;
for (i = 0; i < len; i++) for (i = 0; i < len; i++)
env->insn_aux_data[i].orig_idx = i; env->insn_aux_data[i].orig_idx = i;
env->succ = iarray_realloc(NULL, 2);
if (!env->succ)
goto err_free_env;
env->prog = *prog; env->prog = *prog;
env->ops = bpf_verifier_ops[env->prog->type]; env->ops = bpf_verifier_ops[env->prog->type];
@ -24817,6 +24836,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
bpf_stack_liveness_free(env); bpf_stack_liveness_free(env);
kvfree(env->cfg.insn_postorder); kvfree(env->cfg.insn_postorder);
kvfree(env->scc_info); kvfree(env->scc_info);
kvfree(env->succ);
kvfree(env); kvfree(env);
return ret; return ret;
} }