github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf: Retarget indirect jump targets across prologue prepends

Browse Source 
A few patches do not expand in place, they prepend: bpf_convert_ctx_accesses()
puts the ctx save for gen_epilogue and the instructions of gen_prologue in
front of insn 0, and bpf_do_misc_fixups() prepends the may_goto counter init
at the start of every subprog that uses may_goto.

They copy the original instruction of 'tgt_idx' into the last slot of the
patch buffer, so it now lives at tgt_idx + delta, and call adjust_jmp_off()
to move direct branches from tgt_idx to tgt_idx + delta.

Nothing does the same for indirect branches, so a BPF_MAP_TYPE_INSN_ARRAY
slot that named tgt_idx keeps naming tgt_idx, which is now the first prepended
instruction, and insn_aux_data[tgt_idx].indirect_target makes the JIT emit
the landing pad there. Add a __bpf_patch_insn_data() variant and pass
BPF_PATCH_MOVE_TARGET at the affected call-sites to fix up the delta. The
poke descriptors of direct tail calls are shifted the same way, and the
original instruction is not searched for by content in that mode, as it
is the last slot by construction.

Fixes: 493d9e0d60 ("bpf, x86: add support for indirect jumps")
Fixes: 07ae6c130b ("bpf: Add helper to detect indirect jump targets")
Reported-by: Nicholas Carlini <npc@anthropic.com>
Suggested-by: Nicholas Carlini <npc@anthropic.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Acked-by: Anton Protopopov <a.s.protopopov@gmail.com>
Acked-by: Eduard Zingerman <eddyz87@gmail.com>
Link: https://patch.msgid.link/20260925175244.1136329-11-daniel@iogearbox.net
This commit is contained in:
Daniel Borkmann 2026-09-25 19:52:37 +02:00 committed by Alexei Starovoitov
parent a83512036e
commit fba62d33c1
No known key found for this signature in database
3 changed files with 43 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/bpf.h
View File

@ -4289,7 +4289,7 @@ struct bpf_prog *bpf_prog_find_from_stack(void);
int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog);
int bpf_insn_array_ready(struct bpf_map *map);
void bpf_insn_array_release(struct bpf_map *map);
void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len);
void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len);
void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len);
#ifdef CONFIG_BPF_SYSCALL

4
kernel/bpf/bpf_insn_array.c
View File

@ -236,7 +236,7 @@ void bpf_insn_array_release(struct bpf_map *map)
atomic_set_release(&insn_array->used, 0);
}
void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len)
void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len)
{
struct bpf_insn_array *insn_array = cast_insn_array(map);
int i;
@ -245,7 +245,7 @@ void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len)
return;
for (i = 0; i < map->max_entries; i++) {
if (insn_array->values[i].xlated_off <= off)
if (insn_array->values[i].xlated_off < first)
continue;
if (insn_array->values[i].xlated_off == INSN_DELETED)
continue;

56
kernel/bpf/fixups.c
View File

@ -214,13 +214,19 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env,
}
#endif
enum bpf_patch_mode {
BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */
BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */
};
/* single env->prog->insni[off] instruction was replaced with the range
* insni[off, off + cnt). Adjust corresponding insn_aux_data by copying
* [0, off) and [off, end) to new locations, so the patched range stays zero
*/
static void adjust_insn_aux_data(struct bpf_verifier_env *env,
struct bpf_prog *new_prog, u32 off, u32 cnt,
struct bpf_insn *original_insn)
struct bpf_insn *original_insn,
enum bpf_patch_mode mode)
{
struct bpf_insn_aux_data *data = env->insn_aux_data;
struct bpf_insn *insn = new_prog->insnsi;
@ -253,7 +259,12 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
/* Expand insni[off]'s seen count to the patched range. */
data[i].seen = old_seen;
data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0;
if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) {
/*
* In BPF_PATCH_MOVE_TARGET mode the original insn is the last
* slot by construction.
*/
if (mode == BPF_PATCH_KEEP_TARGET &&
!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) {
data[i].non_stack_access =
data[off + cnt - 1].non_stack_access;
data[off + cnt - 1].non_stack_access = false;
@ -277,9 +288,10 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
* new instructions by the above memmove and memset, but the indirect jump target is
* actually the first instruction, so move it back. This also matches with the behavior
* of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new
* instruction.
* instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one,
* so the flag already sits where needed.
*/
if (data[off + cnt - 1].indirect_target) {
if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) {
data[off].indirect_target = 1;
data[off + cnt - 1].indirect_target = 0;
}
@ -299,7 +311,7 @@ static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len
}
}
static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len)
static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len)
{
int i;
@ -307,7 +319,7 @@ static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len)
return;
for (i = 0; i < env->insn_array_map_cnt; i++)
bpf_insn_array_adjust(env->insn_array_maps[i], off, len);
bpf_insn_array_adjust(env->insn_array_maps[i], first, len);
}
static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len)
@ -318,7 +330,7 @@ static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 of
bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len);
}
static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len)
{
struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
int i, sz = prog->aux->size_poke_tab;
@ -326,7 +338,7 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
for (i = 0; i < sz; i++) {
desc = &tab[i];
if (desc->insn_idx <= off)
if (desc->insn_idx < first)
continue;
desc->insn_idx += len - 1;
}
@ -345,11 +357,14 @@ static bool bpf_rewrite_must_abort(void)
return false;
}
struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
const struct bpf_insn *patch, u32 len)
static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
const struct bpf_insn *patch, u32 len,
enum bpf_patch_mode mode)
{
struct bpf_prog *new_prog;
struct bpf_insn_aux_data *new_data = NULL;
/* The first slot whose targets move. */
u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1;
struct bpf_insn original_insn;
if (bpf_rewrite_must_abort())
@ -375,14 +390,20 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
env->insn_aux_data[off].orig_idx);
return NULL;
}
adjust_insn_aux_data(env, new_prog, off, len, &original_insn);
adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode);
adjust_subprog_starts(env, off, len);
adjust_insn_arrays(env, off, len);
adjust_insn_arrays(env, first, len);
bpf_adjust_func_ptrs(env, off, len);
adjust_poke_descs(new_prog, off, len);
adjust_poke_descs(new_prog, first, len);
return new_prog;
}
struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
const struct bpf_insn *patch, u32 len)
{
return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET);
}
/*
* insn was moved down by delta insns inside its own patch. Operands relative
* to the pc that point in front of the old position did not move with it.
@ -883,7 +904,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
-subprogs[0].stack_depth);
insn_buf[cnt++] = env->prog->insnsi[0];
new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt,
BPF_PATCH_MOVE_TARGET);
if (!new_prog)
return -ENOMEM;
env->prog = new_prog;
@ -906,7 +928,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
verifier_bug(env, "prologue is too long");
return -EFAULT;
} else if (cnt) {
new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt,
BPF_PATCH_MOVE_TARGET);
if (!new_prog)
return -ENOMEM;
@ -2666,7 +2689,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
}
cnt++;
new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt);
new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt,
BPF_PATCH_MOVE_TARGET);
if (!new_prog)
return -ENOMEM;
env->prog = prog = new_prog;