github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-24 23:22:31 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: make bpf_insn_successors to return a pointer

Browse Source 
The bpf_insn_successors() function is used to return successors
to a BPF instruction. So far, an instruction could have 0, 1 or 2
successors. Prepare the verifier code to introduction of instructions
with more than 2 successors (namely, indirect jumps).

To do this, introduce a new struct, struct bpf_iarray, containing
an array of bpf instruction indexes and make bpf_insn_successors
to return a pointer of that type. The storage for all instructions
is allocated in the env->succ, which holds an array of size 2,
to be used for all instructions.

Signed-off-by: Anton Protopopov <a.s.protopopov@gmail.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://lore.kernel.org/r/20251019202145.3944697-10-a.s.protopopov@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Anton Protopopov 2025-10-19 20:21:37 +00:00 committed by Alexei Starovoitov
parent 44481e4925
commit 2f69c56854
3 changed files with 75 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
include/linux/bpf_verifier.h
View File

@ -509,6 +509,15 @@ struct bpf_map_ptr_state {
#define BPF_ALU_SANITIZE (BPF_ALU_SANITIZE_SRC | \
BPF_ALU_SANITIZE_DST)
/*
* An array of BPF instructions.
* Primary usage: return value of bpf_insn_successors.
*/
struct bpf_iarray {
int cnt;
u32 items[];
};
struct bpf_insn_aux_data {
union {
enum bpf_reg_type ptr_type; /* pointer type for load/store insns */
@ -828,6 +837,7 @@ struct bpf_verifier_env {
/* array of pointers to bpf_scc_info indexed by SCC id */
struct bpf_scc_info **scc_info;
u32 scc_cnt;
struct bpf_iarray *succ;
};
static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog)
@ -1050,7 +1060,7 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st
struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off);
int bpf_jmp_offset(struct bpf_insn *insn);
int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2]);
struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx);
void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx);

36
kernel/bpf/liveness.c
View File

@ -34,7 +34,7 @@
* - read and write marks propagation.
* - The propagation phase is a textbook live variable data flow analysis:
*
* state[cc, i].live_after = U [state[cc, s].live_before for s in insn_successors(i)]
* state[cc, i].live_after = U [state[cc, s].live_before for s in bpf_insn_successors(i)]
* state[cc, i].live_before =
* (state[cc, i].live_after / state[cc, i].must_write) U state[i].may_read
*
@ -54,7 +54,7 @@
* The equation for "must_write_acc" propagation looks as follows:
*
* state[cc, i].must_write_acc =
* ∩ [state[cc, s].must_write_acc for s in insn_successors(i)]
* ∩ [state[cc, s].must_write_acc for s in bpf_insn_successors(i)]
* U state[cc, i].must_write
*
* (An intersection of all "must_write_acc" for instruction successors
@ -447,7 +447,12 @@ int bpf_jmp_offset(struct bpf_insn *insn)
__diag_push();
__diag_ignore_all("-Woverride-init", "Allow field initialization overrides for opcode_info_tbl");
inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])
/*
* Returns an array of instructions succ, with succ->items[0], ...,
* succ->items[n-1] with successor instructions, where n=succ->cnt
*/
inline struct bpf_iarray *
bpf_insn_successors(struct bpf_verifier_env *env, u32 idx)
{
static const struct opcode_info {
bool can_jump;
@ -474,19 +479,25 @@ inline int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])
_J(BPF_JSET, {.can_jump = true, .can_fallthrough = true}),
#undef _J
};
struct bpf_prog *prog = env->prog;
struct bpf_insn *insn = &prog->insnsi[idx];
const struct opcode_info *opcode_info;
int i = 0, insn_sz;
struct bpf_iarray *succ;
int insn_sz;
/* pre-allocated array of size up to 2; reset cnt, as it may have been used already */
succ = env->succ;
succ->cnt = 0;
opcode_info = &opcode_info_tbl[BPF_CLASS(insn->code) | BPF_OP(insn->code)];
insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
if (opcode_info->can_fallthrough)
succ[i++] = idx + insn_sz;
succ->items[succ->cnt++] = idx + insn_sz;
if (opcode_info->can_jump)
succ[i++] = idx + bpf_jmp_offset(insn) + 1;
succ->items[succ->cnt++] = idx + bpf_jmp_offset(insn) + 1;
return i;
return succ;
}
__diag_pop();
@ -548,11 +559,12 @@ static inline bool update_insn(struct bpf_verifier_env *env,
struct bpf_insn_aux_data *aux = env->insn_aux_data;
u64 new_before, new_after, must_write_acc;
struct per_frame_masks *insn, *succ_insn;
u32 succ_num, s, succ[2];
struct bpf_iarray *succ;
u32 s;
bool changed;
succ_num = bpf_insn_successors(env->prog, insn_idx, succ);
if (unlikely(succ_num == 0))
succ = bpf_insn_successors(env, insn_idx);
if (succ->cnt == 0)
return false;
changed = false;
@ -564,8 +576,8 @@ static inline bool update_insn(struct bpf_verifier_env *env,
* of successors plus all "must_write" slots of instruction itself.
*/
must_write_acc = U64_MAX;
for (s = 0; s < succ_num; ++s) {
succ_insn = get_frame_masks(instance, frame, succ[s]);
for (s = 0; s < succ->cnt; ++s) {
succ_insn = get_frame_masks(instance, frame, succ->items[s]);
new_after |= succ_insn->live_before;
must_write_acc &= succ_insn->must_write_acc;
}

60
kernel/bpf/verifier.c
View File

@ -17805,6 +17805,22 @@ static int mark_fastcall_patterns(struct bpf_verifier_env *env)
return 0;
}
static struct bpf_iarray *iarray_realloc(struct bpf_iarray *old, size_t n_elem)
{
size_t new_size = sizeof(struct bpf_iarray) + n_elem * sizeof(old->items[0]);
struct bpf_iarray *new;
new = kvrealloc(old, new_size, GFP_KERNEL_ACCOUNT);
if (!new) {
/* this is what callers always want, so simplify the call site */
kvfree(old);
return NULL;
}
new->cnt = n_elem;
return new;
}
/* Visits the instruction at index t and returns one of the following:
* < 0 - an error occurred
* DONE_EXPLORING - the instruction was fully explored
@ -18025,8 +18041,9 @@ static int check_cfg(struct bpf_verifier_env *env)
*/
static int compute_postorder(struct bpf_verifier_env *env)
{
u32 cur_postorder, i, top, stack_sz, s, succ_cnt, succ[2];
u32 cur_postorder, i, top, stack_sz, s;
int *stack = NULL, *postorder = NULL, *state = NULL;
struct bpf_iarray *succ;
postorder = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
state = kvcalloc(env->prog->len, sizeof(int), GFP_KERNEL_ACCOUNT);
@ -18050,11 +18067,11 @@ static int compute_postorder(struct bpf_verifier_env *env)
stack_sz--;
continue;
}
succ_cnt = bpf_insn_successors(env->prog, top, succ);
for (s = 0; s < succ_cnt; ++s) {
if (!state[succ[s]]) {
stack[stack_sz++] = succ[s];
state[succ[s]] |= DISCOVERED;
succ = bpf_insn_successors(env, top);
for (s = 0; s < succ->cnt; ++s) {
if (!state[succ->items[s]]) {
stack[stack_sz++] = succ->items[s];
state[succ->items[s]] |= DISCOVERED;
}
}
state[top] |= EXPLORED;
@ -24313,14 +24330,13 @@ static int compute_live_registers(struct bpf_verifier_env *env)
for (i = 0; i < env->cfg.cur_postorder; ++i) {
int insn_idx = env->cfg.insn_postorder[i];
struct insn_live_regs *live = &state[insn_idx];
int succ_num;
u32 succ[2];
struct bpf_iarray *succ;
u16 new_out = 0;
u16 new_in = 0;
succ_num = bpf_insn_successors(env->prog, insn_idx, succ);
for (int s = 0; s < succ_num; ++s)
new_out |= state[succ[s]].in;
succ = bpf_insn_successors(env, insn_idx);
for (int s = 0; s < succ->cnt; ++s)
new_out |= state[succ->items[s]].in;
new_in = (new_out & ~live->def) | live->use;
if (new_out != live->out || new_in != live->in) {
live->in = new_in;
@ -24373,11 +24389,11 @@ static int compute_scc(struct bpf_verifier_env *env)
const u32 insn_cnt = env->prog->len;
int stack_sz, dfs_sz, err = 0;
u32 *stack, *pre, *low, *dfs;
u32 succ_cnt, i, j, t, w;
u32 i, j, t, w;
u32 next_preorder_num;
u32 next_scc_id;
bool assign_scc;
u32 succ[2];
struct bpf_iarray *succ;
next_preorder_num = 1;
next_scc_id = 1;
@ -24484,12 +24500,12 @@ static int compute_scc(struct bpf_verifier_env *env)
stack[stack_sz++] = w;
}
/* Visit 'w' successors */
succ_cnt = bpf_insn_successors(env->prog, w, succ);
for (j = 0; j < succ_cnt; ++j) {
if (pre[succ[j]]) {
low[w] = min(low[w], low[succ[j]]);
succ = bpf_insn_successors(env, w);
for (j = 0; j < succ->cnt; ++j) {
if (pre[succ->items[j]]) {
low[w] = min(low[w], low[succ->items[j]]);
} else {
dfs[dfs_sz++] = succ[j];
dfs[dfs_sz++] = succ->items[j];
goto dfs_continue;
}
}
@ -24506,8 +24522,8 @@ static int compute_scc(struct bpf_verifier_env *env)
* or if component has a self reference.
*/
assign_scc = stack[stack_sz - 1] != w;
for (j = 0; j < succ_cnt; ++j) {
if (succ[j] == w) {
for (j = 0; j < succ->cnt; ++j) {
if (succ->items[j] == w) {
assign_scc = true;
break;
}
@ -24569,6 +24585,9 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
goto err_free_env;
for (i = 0; i < len; i++)
env->insn_aux_data[i].orig_idx = i;
env->succ = iarray_realloc(NULL, 2);
if (!env->succ)
goto err_free_env;
env->prog = *prog;
env->ops = bpf_verifier_ops[env->prog->type];
@ -24817,6 +24836,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
bpf_stack_liveness_free(env);
kvfree(env->cfg.insn_postorder);
kvfree(env->scc_info);
kvfree(env->succ);
kvfree(env);
return ret;
}