From fba62d33c1254ee706c724e2b3b72fc6e237b7ad Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 25 Sep 2026 19:52:37 +0200 Subject: [PATCH] bpf: Retarget indirect jump targets across prologue prepends A few patches do not expand in place, they prepend: bpf_convert_ctx_accesses() puts the ctx save for gen_epilogue and the instructions of gen_prologue in front of insn 0, and bpf_do_misc_fixups() prepends the may_goto counter init at the start of every subprog that uses may_goto. They copy the original instruction of 'tgt_idx' into the last slot of the patch buffer, so it now lives at tgt_idx + delta, and call adjust_jmp_off() to move direct branches from tgt_idx to tgt_idx + delta. Nothing does the same for indirect branches, so a BPF_MAP_TYPE_INSN_ARRAY slot that named tgt_idx keeps naming tgt_idx, which is now the first prepended instruction, and insn_aux_data[tgt_idx].indirect_target makes the JIT emit the landing pad there. Add a __bpf_patch_insn_data() variant and pass BPF_PATCH_MOVE_TARGET at the affected call-sites to fix up the delta. The poke descriptors of direct tail calls are shifted the same way, and the original instruction is not searched for by content in that mode, as it is the last slot by construction. Fixes: 493d9e0d6083 ("bpf, x86: add support for indirect jumps") Fixes: 07ae6c130b46 ("bpf: Add helper to detect indirect jump targets") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Daniel Borkmann Signed-off-by: Alexei Starovoitov Acked-by: Anton Protopopov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260925175244.1136329-11-daniel@iogearbox.net --- include/linux/bpf.h | 2 +- kernel/bpf/bpf_insn_array.c | 4 +-- kernel/bpf/fixups.c | 56 ++++++++++++++++++++++++++----------- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 8594c8aff745..2cf04bba16bb 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4289,7 +4289,7 @@ struct bpf_prog *bpf_prog_find_from_stack(void); int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog); int bpf_insn_array_ready(struct bpf_map *map); void bpf_insn_array_release(struct bpf_map *map); -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len); +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len); void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len); #ifdef CONFIG_BPF_SYSCALL diff --git a/kernel/bpf/bpf_insn_array.c b/kernel/bpf/bpf_insn_array.c index 61d6a076dfdb..92bbb71576ae 100644 --- a/kernel/bpf/bpf_insn_array.c +++ b/kernel/bpf/bpf_insn_array.c @@ -236,7 +236,7 @@ void bpf_insn_array_release(struct bpf_map *map) atomic_set_release(&insn_array->used, 0); } -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len) { struct bpf_insn_array *insn_array = cast_insn_array(map); int i; @@ -245,7 +245,7 @@ void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len) return; for (i = 0; i < map->max_entries; i++) { - if (insn_array->values[i].xlated_off <= off) + if (insn_array->values[i].xlated_off < first) continue; if (insn_array->values[i].xlated_off == INSN_DELETED) continue; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d4e54fa42dae..37cf130ebb57 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -214,13 +214,19 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, } #endif +enum bpf_patch_mode { + BPF_PATCH_KEEP_TARGET, /* targets of 'off' keep naming 'off', the first new insn */ + BPF_PATCH_MOVE_TARGET, /* targets of 'off' follow the original insn to 'off + len - 1' */ +}; + /* single env->prog->insni[off] instruction was replaced with the range * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, struct bpf_prog *new_prog, u32 off, u32 cnt, - struct bpf_insn *original_insn) + struct bpf_insn *original_insn, + enum bpf_patch_mode mode) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -253,7 +259,12 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; - if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + /* + * In BPF_PATCH_MOVE_TARGET mode the original insn is the last + * slot by construction. + */ + if (mode == BPF_PATCH_KEEP_TARGET && + !memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { data[i].non_stack_access = data[off + cnt - 1].non_stack_access; data[off + cnt - 1].non_stack_access = false; @@ -277,9 +288,10 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, * new instructions by the above memmove and memset, but the indirect jump target is * actually the first instruction, so move it back. This also matches with the behavior * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new - * instruction. + * instruction. For BPF_PATCH_MOVE_TARGET the original instruction is the last one, + * so the flag already sits where needed. */ - if (data[off + cnt - 1].indirect_target) { + if (mode == BPF_PATCH_KEEP_TARGET && data[off + cnt - 1].indirect_target) { data[off].indirect_target = 1; data[off + cnt - 1].indirect_target = 0; } @@ -299,7 +311,7 @@ static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len } } -static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) +static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 first, u32 len) { int i; @@ -307,7 +319,7 @@ static void adjust_insn_arrays(struct bpf_verifier_env *env, u32 off, u32 len) return; for (i = 0; i < env->insn_array_map_cnt; i++) - bpf_insn_array_adjust(env->insn_array_maps[i], off, len); + bpf_insn_array_adjust(env->insn_array_maps[i], first, len); } static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 off, u32 len) @@ -318,7 +330,7 @@ static void adjust_insn_arrays_after_remove(struct bpf_verifier_env *env, u32 of bpf_insn_array_adjust_after_remove(env->insn_array_maps[i], off, len); } -static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) +static void adjust_poke_descs(struct bpf_prog *prog, u32 first, u32 len) { struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab; int i, sz = prog->aux->size_poke_tab; @@ -326,7 +338,7 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) for (i = 0; i < sz; i++) { desc = &tab[i]; - if (desc->insn_idx <= off) + if (desc->insn_idx < first) continue; desc->insn_idx += len - 1; } @@ -345,11 +357,14 @@ static bool bpf_rewrite_must_abort(void) return false; } -struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, - const struct bpf_insn *patch, u32 len) +static struct bpf_prog *__bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len, + enum bpf_patch_mode mode) { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + /* The first slot whose targets move. */ + u32 first = mode == BPF_PATCH_MOVE_TARGET ? off : off + 1; struct bpf_insn original_insn; if (bpf_rewrite_must_abort()) @@ -375,14 +390,20 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len, &original_insn); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn, mode); adjust_subprog_starts(env, off, len); - adjust_insn_arrays(env, off, len); + adjust_insn_arrays(env, first, len); bpf_adjust_func_ptrs(env, off, len); - adjust_poke_descs(new_prog, off, len); + adjust_poke_descs(new_prog, first, len); return new_prog; } +struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len) +{ + return __bpf_patch_insn_data(env, off, patch, len, BPF_PATCH_KEEP_TARGET); +} + /* * insn was moved down by delta insns inside its own patch. Operands relative * to the pc that point in front of the old position did not move with it. @@ -883,7 +904,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1, -subprogs[0].stack_depth); insn_buf[cnt++] = env->prog->insnsi[0]; - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = new_prog; @@ -906,7 +928,8 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env) verifier_bug(env, "prologue is too long"); return -EFAULT; } else if (cnt) { - new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; @@ -2666,7 +2689,8 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) } cnt++; - new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt); + new_prog = __bpf_patch_insn_data(env, subprog_start, insn_buf, cnt, + BPF_PATCH_MOVE_TARGET); if (!new_prog) return -ENOMEM; env->prog = prog = new_prog;