From ad139384ecddb6f3e7e3c3c12765aafae0e39670 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 24 Sep 2026 18:23:53 +0000 Subject: [PATCH 01/11] bpf: Fix overflow of jump offset in constant blinding bpf_jit_blind_insn() rewrites 'if rX op imm goto off' into: AX = imm ^ rnd AX ^= rnd if rX op AX goto off and subtracts 2 from off when the jump is backward. off is s16, so -32767 and -32768 wrap and the jump goes 32767 insns forward, past the end of the prog. The jump is inside the patch, hence bpf_adj_branches() doesn't check it. The kernel crashes when such prog runs with bpf_jit_harden=2. Return -ERANGE in such case. bpf_jit_blind_constants() fails the same way when blinding of other insns moves a jump out of range. Fixes: 4f3446bb809f ("bpf: add generic constant blinding for use in jits") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260924182353.277516-1-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/core.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 2e3bf8113ae9..7f11555a5070 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1362,7 +1362,7 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, { struct bpf_insn *to = to_buff; u32 imm_rnd = get_random_u32(); - s16 off; + int off; BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG); BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); @@ -1438,6 +1438,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, off = from->off; if (off < 0) off -= 2; + if (off < S16_MIN) + return -ERANGE; *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); @@ -1458,6 +1460,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from, off = from->off; if (off < 0) off -= 2; + if (off < S16_MIN) + return -ERANGE; *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, @@ -1606,7 +1610,9 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bp if (!rewritten) continue; - if (env) + if (rewritten < 0) + tmp = ERR_PTR(rewritten); + else if (env) tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten); else tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); From ab39974240a0cff765f3bb9cce81d8001ffdb144 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 25 Sep 2026 08:46:51 +0200 Subject: [PATCH 02/11] bpf: Hold map BTF for the memory allocator destructor record bpf_ma_set_dtor() duplicates map->record so that the bpf_mem_alloc destructor can release the special fields of hash and rhash map elements once the allocator frees them for good. btf_record_dup() only acquires references on kernel and module BTF. Fields whose types live in the map BTF keep pointing into it: a kptr to a local type refers to map->btf, and a list_head or rb_root field carries a value_rec owned by the struct meta table of map->btf. That borrowed state can outlive the map. When RCU callbacks are still in flight, bpf_mem_alloc_destroy() copies the allocator and defers the final drain, together with the destructor context, to a workqueue. bpf_map_free() then frees map->record and drops the map's BTF reference as soon as map_free() returns. Once the program that loaded the BTF is gone as well, the BTF is freed while the worker still uses the duplicate. The worker dereferences it through btf_is_kernel() and btf_find_struct_meta() in bpf_obj_free_fields() when destroying elements that were freed under RCU, and through btf_is_kernel() in btf_record_free() when releasing the context itself: BUG: KASAN: slab-use-after-free in btf_is_kernel+0x19/0x30 Read of size 1 at addr ffff88811505dce8 by task kworker/u16:2/50 Workqueue: events_unbound free_mem_alloc_deferred Call Trace: btf_is_kernel+0x19/0x30 btf_record_free+0xc7/0xf0 htab_dtor_ctx_free+0x1a/0x30 free_mem_alloc_no_barrier+0x39/0x230 free_mem_alloc_deferred+0x2a/0x40 process_scheduled_works+0x5ce/0x9d0 worker_thread+0x42a/0x5e0 kthread+0x1f5/0x230 ret_from_fork+0x1dd/0x390 ret_from_fork_asm+0x1a/0x30 Allocated by task 358: __kmalloc_cache_noprof+0x287/0x510 btf_new_fd+0xf7/0x3d0 __sys_bpf+0x487/0x840 Freed by task 0: kfree+0x186/0x560 rcu_core+0x6f6/0xd40 Freeing a map that still holds more elements than the allocator's high watermark is enough to get there, because the bulk free during teardown queues the RCU callback that makes bpf_mem_alloc_destroy() defer. Non-preallocated hash maps and resizable hash maps both register the destructor and are affected alike. Keep one reference on the map BTF in the destructor context. Every non-kernel pointer in the duplicated record is either map->btf or memory owned by it, so a single reference covers kptrs, list heads, and rb roots, and mirrors what bpf_map_meta_alloc() does for the duplicated record of an inner map. Release it only after the duplicated record has been freed: the worker is preemptible and not in an RCU read-side critical section, so dropping the last reference first would let the BTF be freed while btf_record_free() still reads its fields. Fixes: 1df97a7453ee ("bpf: Register dtor for freeing special fields") Reported-by: Yuan Chen Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260901062845.1379760-3-chenyuan_fl@163.com Link: https://patch.msgid.link/20260925064652.2261885-1-memxor@gmail.com --- kernel/bpf/hashtab.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index f9464e566f10..548eb1cc7c45 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -128,6 +128,7 @@ struct htab_elem { struct htab_btf_record { struct btf_record *record; + struct btf *btf; u32 key_size; }; @@ -497,8 +498,13 @@ static void htab_dtor_ctx_free(void *ctx) { struct htab_btf_record *hrec = ctx; + /* + * The duplicated record still points into the map BTF, so free it + * before dropping the reference that keeps that BTF alive. + */ btf_record_free(hrec->record); - kfree(ctx); + btf_put(hrec->btf); + kfree(hrec); } static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, @@ -521,6 +527,15 @@ static int bpf_ma_set_dtor(struct bpf_map *map, struct bpf_mem_alloc *ma, kfree(hrec); return err; } + /* + * btf_record_dup() only acquires kernel and module BTF. Fields whose + * types live in the map BTF keep pointing into it: kptrs to local + * types refer to map->btf, and graph roots carry a value record owned + * by its struct meta table. The context can outlive the map when the + * allocator defers its teardown, so hold a reference of our own. + */ + hrec->btf = map->btf; + btf_get(hrec->btf); bpf_mem_alloc_set_dtor(ma, dtor, htab_dtor_ctx_free, hrec); return 0; } From 0fb5751a6e411b1b466c06e50021818d9391df98 Mon Sep 17 00:00:00 2001 From: "Florent Revest (Anthropic)" Date: Sat, 26 Sep 2026 13:55:58 +0000 Subject: [PATCH 03/11] bpf: Wait for an RCU tasks grace period before freeing trampoline progs When a prog is detached from a trampoline, it is freed after an RCU grace period, or an RCU tasks trace one if it is sleepable. This covers the tasks that are running the prog, since the prog's enter helper takes the matching read lock before the prog is called. On a preemptible kernel, it doesn't cover a task that was preempted in the trampoline just before the enter helper. That task holds no lock yet, and it calls the prog after it was freed: BUG: KASAN: vmalloc-out-of-bounds in __bpf_prog_enter_recur+0x3a5/0x3f0 Read of size 8 at addr ffffc90000055040 by task candidate/110 CPU: 1 UID: 0 PID: 110 Comm: candidate Not tainted 7.3.0-rc2-00014-g15071f2a1263-dirty #2 PREEMPT(full) Call Trace: __bpf_prog_enter_recur+0x3a5/0x3f0 bpf_trampoline_6442509193+0x37/0xf1 __x64_sys_futex+0x9/0x410 do_syscall_64+0xb0/0x530 ... Wait for an RCU tasks grace period before the existing one when freeing a prog that was linked to a trampoline. An RCU tasks grace period only ends once the tasks that were preempted have run again, and bpf_tramp_image_put() already relies on it to free the image. It doesn't wait for tasks that sleep in the trampoline, the next commit takes care of those. Only progs that were linked to a trampoline can be called this way, so bpf_trampoline_add_prog() marks them and other progs are still freed as before. Fixes: e21aa341785c ("bpf: Fix fexit trampoline.") Reported-by: Junseo Lim Signed-off-by: Florent Revest (Anthropic) Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926135605.1217928-2-florent.revest@linux.dev Closes: https://lore.kernel.org/bpf/aqdrwVpanH3WGurX@omen-arch/ --- include/linux/bpf.h | 1 + kernel/bpf/syscall.c | 19 ++++++++++++++++++- kernel/bpf/trampoline.c | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1d2676782d70..a8ec85127571 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1770,6 +1770,7 @@ struct bpf_prog_aux { bool offload_requested; /* Program is bound and offloaded to the netdev. */ bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ + bool tramp_linked; /* true if it was ever linked to a trampoline */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 244a939b9d2d..96217b99399d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2448,6 +2448,21 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) bpf_prog_free(aux->prog); } +/* + * Progs called from a trampoline can also be reached by a task that was + * preempted in the trampoline before the prog's enter helper took its RCU + * read lock, wait for those first. + */ +static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) +{ + struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog; + + if (prog->sleepable) + call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu); + else + call_rcu(rcu, __bpf_prog_put_rcu); +} + static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { bpf_prog_kallsyms_del_all(prog); @@ -2461,7 +2476,9 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) btf_put(prog->aux->attach_btf); if (deferred) { - if (prog->sleepable) + if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked) + call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks); + else if (prog->sleepable) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 90b70ea0d370..9d69c066a817 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -907,6 +907,7 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, } hlist_add_head(&node->tramp_hlist, prog_list); + node->link->prog->aux->tramp_linked = true; if (kind == BPF_TRAMP_FSESSION) { tr->progs_cnt[BPF_TRAMP_FENTRY]++; fexit = fsession_exit(node); From 17637e1a581a22466ac3620a91e099683ff9cc6f Mon Sep 17 00:00:00 2001 From: "Florent Revest (Anthropic)" Date: Sat, 26 Sep 2026 13:55:59 +0000 Subject: [PATCH 04/11] bpf: Skip detached progs in trampoline images that are still in use bpf_tramp_image_put() makes sure a trampoline image is not freed while a task may still be running in it, but nothing similar is done for the progs called by that image. Detach drops the last prog reference right away and the prog is freed after grace periods, on the basis that a task still in the traced function skips the fexit progs once the nop at ip_after_call is patched to a jump. That leaves out a task sleeping in a sleepable prog that runs before the detached one, which no grace period waits for: CPU 0 CPU 1 in image I, sleeping in prog S detach P from I's trampoline -> new image, bpf_tramp_image_put(I) bpf_prog_put(P), last ref grace periods, P freed back from S __bpf_prog_enter(P) call P->bpf_func If S and P are fexit progs the task is already past the patched jump, and fentry only images don't have one. On x86 this is an int3 in poisoned bpf_prog_pack memory: Oops: int3: 0000 [#1] SMP NOPTI CPU: 18 UID: 0 PID: 94573 Comm: x169 Not tainted 6.18.44 #1 PREEMPT(lazy) RIP: 0010:0xffffffffc0601d8d Call Trace: ? bpf_trampoline_6442515411+0x1a4/0x21b bpf_lsm_bprm_committed_creds+0x5/0x10 security_bprm_committed_creds+0x5f/0x70 begin_new_exec+0x2d6/0x410 ... We hit this in production when progs attached through trampolines got detached while their hooks were busy, and it was independently found with a fuzzer and KASAN. Have the JITs emit a patchable nop in front of each prog call sequence and record it in the image. When a prog is detached, patch its nop to a jump over the call sequence. Progs that stay attached keep running for the tasks that are in the image, and ip_after_call isn't needed anymore. The task can be in any image that isn't freed yet, not only in the current one. It sleeps in image I1 that calls S, P and Q, then P is detached and the trampoline moves to image I2, then Q is detached and its call is still in I1. So the trampoline keeps a list of its images until they are freed, and detaching a prog patches its nop in all of them. Images hold a reference on the trampoline for that long. On riscv and loongarch a jump of any range takes several instructions, and a task preempted in the middle of them could resume into half of the new sequence. The nop is a single instruction there, patched to a near branch through arch_bpf_trampoline_skip(). With the extra nops, BPF_MAX_TRAMP_LINKS progs no longer fit in a page on x86 and arm64 (and already didn't on powerpc), so lower the limit there like s390 does. Fixes: e21aa341785c ("bpf: Fix fexit trampoline.") Reported-by: Sechang Lim Suggested-by: Alexei Starovoitov Signed-off-by: Florent Revest (Anthropic) Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926135605.1217928-3-florent.revest@linux.dev Closes: https://lore.kernel.org/bpf/20260815071927.147049-1-zirajs7@gmail.com/ --- arch/arm64/net/bpf_jit_comp.c | 37 +++++++++------ arch/loongarch/net/bpf_jit.c | 60 +++++++++++++++-------- arch/powerpc/net/bpf_jit_comp.c | 45 +++++++++--------- arch/riscv/net/bpf_jit_comp64.c | 56 +++++++++++++++------- arch/s390/net/bpf_jit_comp.c | 46 +++++++++++------- arch/x86/net/bpf_jit_comp.c | 26 ++++++---- include/linux/bpf.h | 44 +++++++++++++++-- kernel/bpf/trampoline.c | 84 ++++++++++++++++++++++++++------- 8 files changed, 277 insertions(+), 121 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c5f55d6161fe..c1279fabce47 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2418,10 +2418,11 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } -static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, - int bargs_off, int retval_off, int run_ctx_off, - bool save_ret) +static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_node *node, int bargs_off, + int retval_off, int run_ctx_off, bool save_ret) { + void *skip; __le32 *branch; u64 enter_prog; u64 exit_prog; @@ -2431,6 +2432,10 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, enter_prog = (u64)bpf_trampoline_enter(p); exit_prog = (u64)bpf_trampoline_exit(p); + /* nop, patched to skip this prog when it is detached */ + skip = ctx->ro_image + ctx->idx; + emit(A64_NOP, ctx); + if (node->cookie == 0) { /* if cookie is zero, one instruction is enough to store it */ emit(A64_STR64I(A64_ZR, A64_SP, run_ctx_off + cookie_off), ctx); @@ -2483,11 +2488,13 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, emit(A64_ADD_I(1, A64_R(2), A64_SP, run_ctx_off), ctx); emit_call(exit_prog, ctx); + + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_image + ctx->idx); } -static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, - int bargs_off, int retval_off, int run_ctx_off, - __le32 **branches) +static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_nodes *tn, int bargs_off, + int retval_off, int run_ctx_off, __le32 **branches) { int i; @@ -2496,7 +2503,7 @@ static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, */ emit(A64_STR64I(A64_ZR, A64_SP, retval_off), ctx); for (i = 0; i < tn->nr_nodes; i++) { - invoke_bpf_prog(ctx, tn->nodes[i], bargs_off, retval_off, + invoke_bpf_prog(ctx, im, tn->nodes[i], bargs_off, retval_off, run_ctx_off, true); /* if (*(u64 *)(sp + retval_off) != 0) * goto do_fexit; @@ -2882,7 +2889,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, meta, func_meta_off); cookie_bargs_off--; } - invoke_bpf_prog(ctx, fentry->nodes[i], bargs_off, + invoke_bpf_prog(ctx, im, fentry->nodes[i], bargs_off, retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET); } @@ -2893,7 +2900,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (!branches) return -ENOMEM; - invoke_bpf_mod_ret(ctx, fmod_ret, bargs_off, retval_off, + invoke_bpf_mod_ret(ctx, im, fmod_ret, bargs_off, retval_off, run_ctx_off, branches); } @@ -2906,9 +2913,6 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, emit(A64_RET(A64_R(10)), ctx); /* store return value */ emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx); - /* reserve a nop for bpf_tramp_image_put */ - im->ip_after_call = ctx->ro_image + ctx->idx; - emit(A64_NOP, ctx); } /* update the branches saved in invoke_bpf_mod_ret with cbnz */ @@ -2930,12 +2934,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, meta, func_meta_off); cookie_bargs_off--; } - invoke_bpf_prog(ctx, fexit->nodes[i], bargs_off, retval_off, + invoke_bpf_prog(ctx, im, fexit->nodes[i], bargs_off, retval_off, run_ctx_off, false); } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_image + ctx->idx; /* for the first pass, assume the worst case */ if (!ctx->image) ctx->idx += 4; @@ -2994,7 +2997,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, .image = NULL, .idx = 0, }; - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct arg_aux aaux; int ret; @@ -3281,6 +3284,10 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, * longer reachable, since bpf_tramp_image_put() function already * uses percpu_ref and task-based rcu to do the sync, no need to call * the sync version here, see bpf_tramp_image_put() for details. + * + * 3. when a detached prog is patched out of a trampoline, a CPU that + * still executes the old nop calls the prog before it went through + * a quiescent state, and the prog is freed after grace periods. */ ret = aarch64_insn_patch_text_nosync(ip, new_insn); out: diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 4da278900938..e78eb582c300 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1578,6 +1578,24 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len) return ret ? ERR_PTR(-EINVAL) : dst; } +int arch_bpf_trampoline_skip(void *nop, void *target) +{ + u32 old_insn = INSN_NOP; + u32 new_insn = larch_insn_gen_b((unsigned long)nop, (unsigned long)target); + int ret; + + if (memcmp(nop, &old_insn, LOONGARCH_INSN_SIZE)) + return -EFAULT; + + cpus_read_lock(); + mutex_lock(&text_mutex); + ret = larch_insn_text_copy(nop, &new_insn, LOONGARCH_INSN_SIZE); + mutex_unlock(&text_mutex); + cpus_read_unlock(); + + return ret; +} + int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr) @@ -1696,13 +1714,18 @@ static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off, } } -static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n, - int args_off, int retval_off, int run_ctx_off, bool save_ret) +static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_node *n, int args_off, int retval_off, + int run_ctx_off, bool save_ret) { int ret; u32 *branch; struct bpf_prog *p = n->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + void *skip = ctx->ro_image + ctx->idx; + + /* nop, patched to a b over this prog when it is detached */ + emit_insn(ctx, nop); if (n->cookie) emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, @@ -1755,13 +1778,17 @@ static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n, /* arg3: &run_ctx */ emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); + if (ret) + return ret; - return ret; + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_image + ctx->idx); + return 0; } -static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, - int args_off, int retval_off, int run_ctx_off, - int func_meta_off, bool save_ret, u64 func_meta, int cookie_off) +static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_nodes *tn, int args_off, int retval_off, + int run_ctx_off, int func_meta_off, bool save_ret, + u64 func_meta, int cookie_off) { int i, cur_cookie = (cookie_off - args_off) / 8; @@ -1774,7 +1801,8 @@ static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, meta); cur_cookie--; } - err = invoke_bpf_prog(ctx, tn->nodes[i], args_off, retval_off, run_ctx_off, save_ret); + err = invoke_bpf_prog(ctx, im, tn->nodes[i], args_off, retval_off, + run_ctx_off, save_ret); if (err) return err; } @@ -2017,7 +2045,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i } if (fentry->nr_nodes) { - ret = invoke_bpf(ctx, fentry, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(ctx, im, fentry, args_off, retval_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off); if (ret) return ret; @@ -2029,7 +2057,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); for (i = 0; i < fmod_ret->nr_nodes; i++) { - ret = invoke_bpf_prog(ctx, fmod_ret->nodes[i], + ret = invoke_bpf_prog(ctx, im, fmod_ret->nodes[i], args_off, retval_off, run_ctx_off, true); if (ret) goto out; @@ -2051,10 +2079,6 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i goto out; emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); - im->ip_after_call = ctx->ro_image + ctx->idx; - /* Reserve space for the move_imm + jirl instruction */ - for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) - emit_insn(ctx, nop); } for (i = 0; ctx->image && i < fmod_ret->nr_nodes; i++) { @@ -2068,14 +2092,13 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); if (fexit->nr_nodes) { - ret = invoke_bpf(ctx, fexit, args_off, retval_off, run_ctx_off, + ret = invoke_bpf(ctx, im, fexit, args_off, retval_off, run_ctx_off, func_meta_off, false, func_meta, cookie_off); if (ret) goto out; } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_image + ctx->idx; move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); ret = emit_call(ctx, (const u64)__bpf_tramp_exit); if (ret) @@ -2177,11 +2200,8 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { int ret; - struct jit_ctx ctx; - struct bpf_tramp_image im; - - ctx.image = NULL; - ctx.idx = 0; + struct jit_ctx ctx = {}; + struct bpf_tramp_image im = {}; ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tnodes, func_addr, flags); diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7b07b43575f1..7a612688e7a2 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -602,14 +602,18 @@ int arch_protect_bpf_trampoline(void *image, unsigned int size) } static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx, - struct bpf_tramp_node *n, int regs_off, int retval_off, - int run_ctx_off, bool save_ret) + struct bpf_tramp_image *im, struct bpf_tramp_node *n, + int regs_off, int retval_off, int run_ctx_off, bool save_ret) { struct bpf_prog *p = n->link->prog; ppc_inst_t branch_insn; - u32 jmp_idx; + u32 jmp_idx, skip_idx; int ret = 0; + /* nop, patched to skip this prog when it is detached */ + skip_idx = ctx->idx; + EMIT(PPC_RAW_NOP()); + /* Save cookie */ if (IS_ENABLED(CONFIG_PPC64)) { PPC_LI64(_R3, n->cookie); @@ -679,13 +683,17 @@ static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ct EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off)); ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, (unsigned long)bpf_trampoline_exit(p)); + if (ret) + return ret; - return ret; + if (ro_image) /* image is NULL for dummy pass */ + bpf_tramp_image_add_skip(im, p, &ro_image[skip_idx], &ro_image[ctx->idx]); + return 0; } static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx, - struct bpf_tramp_nodes *tn, int regs_off, int retval_off, - int run_ctx_off, u32 *branches) + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, + int regs_off, int retval_off, int run_ctx_off, u32 *branches) { int i; @@ -696,8 +704,8 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context EMIT(PPC_RAW_LI(_R3, 0)); EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); for (i = 0; i < tn->nr_nodes; i++) { - if (invoke_bpf_prog(image, ro_image, ctx, tn->nodes[i], regs_off, retval_off, - run_ctx_off, true)) + if (invoke_bpf_prog(image, ro_image, ctx, im, tn->nodes[i], regs_off, + retval_off, run_ctx_off, true)) return -EINVAL; /* @@ -1043,8 +1051,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im cookie_ctx_off--; } - if (invoke_bpf_prog(image, ro_image, ctx, fentry->nodes[i], regs_off, retval_off, - run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) + if (invoke_bpf_prog(image, ro_image, ctx, im, fentry->nodes[i], regs_off, + retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) return -EINVAL; } @@ -1053,7 +1061,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (!branches) return -ENOMEM; - if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off, + if (invoke_bpf_mod_ret(image, ro_image, ctx, im, fmod_ret, regs_off, retval_off, run_ctx_off, branches)) { ret = -EINVAL; goto cleanup; @@ -1090,11 +1098,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Restore updated tail_call_cnt */ if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off); - - /* Reserve space to patch branch instruction to skip fexit progs */ - if (ro_image) /* image is NULL for dummy pass */ - im->ip_after_call = &((u32 *)ro_image)[ctx->idx]; - EMIT(PPC_RAW_NOP()); } /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */ @@ -1123,16 +1126,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im cookie_ctx_off--; } - if (invoke_bpf_prog(image, ro_image, ctx, fexit->nodes[i], regs_off, retval_off, - run_ctx_off, false)) { + if (invoke_bpf_prog(image, ro_image, ctx, im, fexit->nodes[i], regs_off, + retval_off, run_ctx_off, false)) { ret = -EINVAL; goto cleanup; } } if (flags & BPF_TRAMP_F_CALL_ORIG) { - if (ro_image) /* image is NULL for dummy pass */ - im->ip_epilogue = &((u32 *)ro_image)[ctx->idx]; PPC_LI_ADDR(_R3, im); ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, (unsigned long)__bpf_tramp_exit); @@ -1192,7 +1193,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; int ret; ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr); @@ -1320,7 +1321,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, /* * If we are not poking at bpf prog entry, then we are simply patching in/out - * an unconditional branch instruction at im->ip_after_call + * an unconditional branch instruction in a trampoline image */ if (offset) { if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) { diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 151031e97a24..01fe66774f02 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -822,6 +822,24 @@ static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); } +int arch_bpf_trampoline_skip(void *nop, void *target) +{ + u32 old_insn = rv_nop(); + u32 new_insn = rv_jal(RV_REG_ZERO, (target - nop) >> 1); + int ret; + + if (memcmp(nop, &old_insn, sizeof(old_insn))) + return -EFAULT; + + cpus_read_lock(); + mutex_lock(&text_mutex); + ret = patch_text(nop, &new_insn, sizeof(new_insn)); + mutex_unlock(&text_mutex); + cpus_read_unlock(); + + return ret; +} + int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr) @@ -904,12 +922,17 @@ static void emit_store_stack_imm64(u8 reg, int stack_off, u64 imm64, emit_sd(RV_REG_FP, stack_off, reg, ctx); } -static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval_off, - int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) +static int invoke_bpf_prog(struct bpf_tramp_image *im, struct bpf_tramp_node *node, + int args_off, int retval_off, int run_ctx_off, bool save_ret, + struct rv_jit_context *ctx) { int ret, branch_off; struct bpf_prog *p = node->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + void *skip = ctx->ro_insns + ctx->ninsns; + + /* nop, patched to a jal over this prog when it is detached */ + emit(rv_nop(), ctx); if (node->cookie) emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, node->cookie, ctx); @@ -962,13 +985,17 @@ static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval /* arg3: &run_ctx */ emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); + if (ret) + return ret; - return ret; + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_insns + ctx->ninsns); + return 0; } -static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off, - int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta, - int cookie_off, struct rv_jit_context *ctx) +static int invoke_bpf(struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, + int args_off, int retval_off, int run_ctx_off, int func_meta_off, + bool save_ret, u64 func_meta, int cookie_off, + struct rv_jit_context *ctx) { int i, cur_cookie = (cookie_off - args_off) / 8; @@ -981,8 +1008,8 @@ static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off, emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx); cur_cookie--; } - err = invoke_bpf_prog(tn->nodes[i], args_off, retval_off, run_ctx_off, - save_ret, ctx); + err = invoke_bpf_prog(im, tn->nodes[i], args_off, retval_off, + run_ctx_off, save_ret, ctx); if (err) return err; } @@ -1170,7 +1197,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (fentry->nr_nodes) { - ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(im, fentry, args_off, retval_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx); if (ret) return ret; @@ -1184,7 +1211,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* cleanup to avoid garbage return value confusion */ emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); for (i = 0; i < fmod_ret->nr_nodes; i++) { - ret = invoke_bpf_prog(fmod_ret->nodes[i], args_off, retval_off, + ret = invoke_bpf_prog(im, fmod_ret->nodes[i], args_off, retval_off, run_ctx_off, true, ctx); if (ret) goto out; @@ -1211,10 +1238,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); - im->ip_after_call = ctx->ro_insns + ctx->ninsns; - /* 2 nops reserved for auipc+jalr pair */ - emit(rv_nop(), ctx); - emit(rv_nop(), ctx); } /* update branches saved in invoke_bpf_mod_ret with bnez */ @@ -1230,14 +1253,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); if (fexit->nr_nodes) { - ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(im, fexit, args_off, retval_off, run_ctx_off, func_meta_off, false, func_meta, cookie_off, ctx); if (ret) goto out; } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_insns + ctx->ninsns; emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); if (ret) @@ -1299,7 +1321,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct rv_jit_context ctx; int ret; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index c4b47070bb59..1b2566012b63 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2566,6 +2566,8 @@ struct bpf_tramp_jit { int r14_off; /* Offset of saved %r14, has to be at the * bottom */ int do_fexit; /* do_fexit: label */ + int skip[BPF_MAX_TRAMP_LINKS]; /* skip: labels after each prog */ + int nr_progs; }; static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) @@ -2584,6 +2586,7 @@ static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_o } static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, + struct bpf_tramp_image *im, const struct btf_func_model *m, struct bpf_tramp_node *node, bool save_ret) { @@ -2591,8 +2594,20 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, int cookie_off = tjit->run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie); struct bpf_prog *p = node->link->prog; + void *skip = jit->prg_buf + jit->prg; + int idx = tjit->nr_progs++; int patch; + if (idx >= ARRAY_SIZE(tjit->skip)) + return -E2BIG; + + /* + * nop, patched to skip this prog when it is detached + */ + + /* brcl 0,skip */ + EMIT6_PCREL_RILC(0xc0040000, 0, tjit->skip[idx]); + /* * run_ctx.cookie = node->cookie; */ @@ -2652,10 +2667,15 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, /* brasl %r14,__bpf_prog_exit */ EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_exit(p)); + /* skip: */ + tjit->skip[idx] = jit->prg; + bpf_tramp_image_add_skip(im, p, skip, jit->prg_buf + jit->prg); + return 0; } static int invoke_bpf(struct bpf_tramp_jit *tjit, + struct bpf_tramp_image *im, const struct btf_func_model *m, struct bpf_tramp_nodes *tn, bool save_ret, u64 func_meta, int cookie_off) @@ -2670,7 +2690,7 @@ static int invoke_bpf(struct bpf_tramp_jit *tjit, emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta); cur_cookie--; } - if (invoke_bpf_prog(tjit, m, tn->nodes[i], save_ret)) + if (invoke_bpf_prog(tjit, im, m, tn->nodes[i], save_ret)) return -EINVAL; } @@ -2712,6 +2732,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, u64 func_meta; int i, j; + /* The skip labels are taken from the previous pass. */ + tjit->nr_progs = 0; + if (im) + im->nr_skips = 0; + /* Support as many stack arguments as "mvc" instruction can handle. */ nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS); nr_stack_args = m->nr_args - nr_reg_args; @@ -2875,7 +2900,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0); } - if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, + if (invoke_bpf(tjit, im, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off)) return -EINVAL; @@ -2889,7 +2914,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 0xf000 | tjit->retval_off); for (i = 0; i < fmod_ret->nr_nodes; i++) { - if (invoke_bpf_prog(tjit, m, fmod_ret->nodes[i], true)) + if (invoke_bpf_prog(tjit, im, m, fmod_ret->nodes[i], true)) return -EINVAL; /* @@ -2943,15 +2968,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* mvc tccnt_off(%r15),tail_call_cnt(4,%r15) */ _EMIT6(0xd203f000 | tjit->tccnt_off, 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); - - im->ip_after_call = jit->prg_buf + jit->prg; - - /* - * The following nop will be patched by bpf_tramp_image_put(). - */ - - /* brcl 0,im->ip_epilogue */ - EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); } /* Set the "is_return" flag for fsession. */ @@ -2962,12 +2978,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* do_fexit: */ tjit->do_fexit = jit->prg; - if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off)) + if (invoke_bpf(tjit, im, m, fexit, false, func_meta, cookie_off)) return -EINVAL; if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = jit->prg_buf + jit->prg; - /* * __bpf_tramp_exit(im); */ @@ -3016,7 +3030,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *orig_call) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct bpf_tramp_jit tjit; int ret; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2853e87797a7..6bca87457e87 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3217,16 +3217,21 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, } static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_node *node, int stack_size, int run_ctx_off, bool save_ret, void *image, void *rw_image) { u8 *prog = *pprog; - u8 *jmp_insn; + u8 *jmp_insn, *skip; int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); struct bpf_prog *p = node->link->prog; u64 cookie = node->cookie; + /* nop, patched to skip this prog when it is detached */ + skip = image + (prog - (u8 *)rw_image); + emit_nops(&prog, X86_PATCH_SIZE); + /* mov rdi, cookie */ emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie); @@ -3301,6 +3306,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, if (emit_rsb_call(&prog, bpf_trampoline_exit(p), image + (prog - (u8 *)rw_image))) return -EINVAL; + bpf_tramp_image_add_skip(im, p, skip, image + (prog - (u8 *)rw_image)); + *pprog = prog; return 0; } @@ -3332,6 +3339,7 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) } static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tl, int stack_size, int run_ctx_off, int func_meta_off, bool save_ret, void *image, void *rw_image, u64 func_meta, @@ -3346,7 +3354,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, func_meta | (cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT)); cur_cookie--; } - if (invoke_bpf_prog(m, &prog, tl->nodes[i], stack_size, + if (invoke_bpf_prog(m, &prog, im, tl->nodes[i], stack_size, run_ctx_off, save_ret, image, rw_image)) return -EINVAL; } @@ -3355,6 +3363,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, } static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tl, int stack_size, int run_ctx_off, u8 **branches, void *image, void *rw_image) @@ -3368,7 +3377,7 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, emit_mov_imm32(&prog, false, BPF_REG_0, 0); emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); for (i = 0; i < tl->nr_nodes; i++) { - if (invoke_bpf_prog(m, &prog, tl->nodes[i], stack_size, run_ctx_off, true, + if (invoke_bpf_prog(m, &prog, im, tl->nodes[i], stack_size, run_ctx_off, true, image, rw_image)) return -EINVAL; @@ -3640,7 +3649,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (fentry->nr_nodes) { - if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, func_meta_off, + if (invoke_bpf(m, &prog, im, fentry, regs_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, image, rw_image, func_meta, cookie_off)) return -EINVAL; @@ -3652,7 +3661,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (!branches) return -ENOMEM; - if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off, + if (invoke_bpf_mod_ret(m, &prog, im, fmod_ret, regs_off, run_ctx_off, branches, image, rw_image)) { ret = -EINVAL; goto cleanup; @@ -3682,8 +3691,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } /* remember return value in a stack for bpf prog to access */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); - im->ip_after_call = image + (prog - (u8 *)rw_image); - emit_nops(&prog, X86_PATCH_SIZE); } if (fmod_ret->nr_nodes) { @@ -3708,7 +3715,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta); if (fexit->nr_nodes) { - if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, func_meta_off, + if (invoke_bpf(m, &prog, im, fexit, regs_off, run_ctx_off, func_meta_off, false, image, rw_image, func_meta, cookie_off)) { ret = -EINVAL; goto cleanup; @@ -3723,7 +3730,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * restored to R0. */ if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = image + (prog - (u8 *)rw_image); /* arg1: mov rdi, im */ emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); if (emit_rsb_call(&prog, __bpf_tramp_exit, image + (prog - (u8 *)rw_image))) { @@ -3811,7 +3817,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; void *image; int ret; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a8ec85127571..0ecb9418dfbd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1258,11 +1258,15 @@ struct btf_func_model { #define BPF_TRAMP_F_INDIRECT BIT(8) /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 - * bytes on x86. + * bytes on x86. The trampoline image has to fit in PAGE_SIZE. */ enum { -#if defined(__s390x__) +#if defined(__s390x__) || defined(__powerpc64__) BPF_MAX_TRAMP_LINKS = 27, +#elif defined(__x86_64__) + BPF_MAX_TRAMP_LINKS = 36, +#elif defined(__aarch64__) + BPF_MAX_TRAMP_LINKS = 37, #else BPF_MAX_TRAMP_LINKS = 38, #endif @@ -1315,6 +1319,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i void *arch_alloc_bpf_trampoline(unsigned int size); void arch_free_bpf_trampoline(void *image, unsigned int size); int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size); +int arch_bpf_trampoline_skip(void *nop, void *target); int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr); @@ -1363,19 +1368,48 @@ enum bpf_tramp_prog_type { BPF_TRAMP_FSESSION, }; +/* + * Each prog call in a trampoline image is preceded by a nop. When the prog is + * detached, the nop is patched to a jump to target, right after the call, so + * that tasks still running in the image skip the prog. + */ +struct bpf_tramp_skip { + struct bpf_prog *prog; + void *nop; + void *target; +}; + struct bpf_tramp_image { void *image; int size; struct bpf_ksym ksym; struct percpu_ref pcref; - void *ip_after_call; - void *ip_epilogue; + bool call_orig; + /* entry in tr->images, the image holds a reference on tr */ + struct bpf_trampoline *tr; + struct list_head list; + int nr_skips; + struct bpf_tramp_skip *skips; union { struct rcu_head rcu; struct work_struct work; }; }; +static inline void bpf_tramp_image_add_skip(struct bpf_tramp_image *im, struct bpf_prog *prog, + void *nop, void *target) +{ + struct bpf_tramp_skip *skip; + + /* struct_ops trampolines and arch_bpf_trampoline_size() have no image */ + if (!im || !im->skips) + return; + skip = &im->skips[im->nr_skips++]; + skip->prog = prog; + skip->nop = nop; + skip->target = target; +} + struct bpf_trampoline { /* hlist for trampoline_key_table */ struct hlist_node hlist_key; @@ -1402,6 +1436,8 @@ struct bpf_trampoline { int progs_cnt[BPF_TRAMP_MAX]; /* Executable image of trampoline */ struct bpf_tramp_image *cur_image; + /* Images not freed yet, cur_image and older ones still in use */ + struct list_head images; /* Used as temporary old image storage for multi_attach */ struct { struct bpf_tramp_image *old_image; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 9d69c066a817..bf4cb3dd444d 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -401,6 +401,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); + INIT_LIST_HEAD(&tr->images); for (i = 0; i < BPF_TRAMP_MAX; i++) INIT_HLIST_HEAD(&tr->progs_hlist[i]); out: @@ -565,15 +566,22 @@ static void bpf_tramp_image_free(struct bpf_tramp_image *im) arch_free_bpf_trampoline(im->image, im->size); bpf_jit_uncharge_modmem(im->size); percpu_ref_exit(&im->pcref); + kfree(im->skips); kfree_rcu(im, rcu); } static void __bpf_tramp_image_put_deferred(struct work_struct *work) { struct bpf_tramp_image *im; + struct bpf_trampoline *tr; im = container_of(work, struct bpf_tramp_image, work); + tr = im->tr; + trampoline_lock(tr); + list_del(&im->list); + trampoline_unlock(tr); bpf_tramp_image_free(im); + bpf_trampoline_put(tr); } /* callback, fexit step 3 or fentry step 2 */ @@ -601,7 +609,7 @@ static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu) struct bpf_tramp_image *im; im = container_of(rcu, struct bpf_tramp_image, rcu); - if (im->ip_after_call) + if (im->call_orig) /* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */ percpu_ref_kill(&im->pcref); else @@ -621,9 +629,9 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * * The trampoline is unreachable before bpf_tramp_image_put(). * - * First, patch the trampoline to avoid calling into fexit progs. - * The progs will be freed even if the original function is still - * executing or sleeping. + * Progs are patched out of the image when they are detached, see + * bpf_trampoline_skip_prog(), so they can be freed even if a task is + * still in the image. * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on * first few asm instructions to execute and call into * __bpf_tramp_enter->percpu_ref_get. @@ -637,11 +645,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * percpu_ref_kill will be waiting for. Hence the first * call_rcu_tasks() is not necessary. */ - if (im->ip_after_call) { - int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - im->ip_epilogue); - WARN_ON(err); + if (im->call_orig) { if (IS_ENABLED(CONFIG_TASKS_RCU)) call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks); else @@ -658,7 +662,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); } -static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) +static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size, int nr_progs) { struct bpf_tramp_image *im; struct bpf_ksym *ksym; @@ -669,6 +673,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) if (!im) goto out; + im->skips = kzalloc_objs(*im->skips, nr_progs); + if (!im->skips) + goto out_free_im; + err = bpf_jit_charge_modmem(size); if (err) goto out_free_im; @@ -695,6 +703,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) out_uncharge: bpf_jit_uncharge_modmem(size); out_free_im: + kfree(im->skips); kfree(im); out: return ERR_PTR(err); @@ -771,11 +780,12 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut goto out; } - im = bpf_tramp_image_alloc(tr->key, size); + im = bpf_tramp_image_alloc(tr->key, size, total); if (IS_ERR(im)) { err = PTR_ERR(im); goto out; } + im->call_orig = tr->flags & BPF_TRAMP_F_CALL_ORIG; err = arch_prepare_bpf_trampoline(im, im->image, im->image + size, &tr->func.model, tr->flags, tnodes, @@ -806,8 +816,14 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut #endif out_free: - if (err) + if (err) { bpf_tramp_image_free(im); + } else { + /* track the image until it is freed, for bpf_trampoline_skip_prog() */ + refcount_inc(&tr->refcnt); + im->tr = tr; + list_add(&im->list, &tr->images); + } out: /* If any error happens, restore previous flags */ if (err) @@ -921,6 +937,41 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, return 0; } +/* + * Patch the nop in front of a prog call to a jump over it. A task can be + * preempted anywhere in the image, so archs that need several instructions for + * a jump of any range patch a single near branch here instead. + */ +int __weak arch_bpf_trampoline_skip(void *nop, void *target) +{ + return bpf_arch_text_poke(nop, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, target); +} + +/* + * prog was detached and can be freed, but tasks may still be running in images + * that call it, sleeping in an earlier prog for example. They can be in any + * image that is not freed yet, not only in cur_image, so patch all of them to + * jump over prog. + */ +static void bpf_trampoline_skip_prog(struct bpf_trampoline *tr, struct bpf_prog *prog) +{ + struct bpf_tramp_image *im; + int i, err; + + list_for_each_entry(im, &tr->images, list) { + for (i = 0; i < im->nr_skips; i++) { + struct bpf_tramp_skip *skip = &im->skips[i]; + + if (skip->prog != prog) + continue; + err = arch_bpf_trampoline_skip(skip->nop, skip->target); + WARN_ON_ONCE(err); + /* not a nop anymore, and prog's address can be reused */ + skip->prog = NULL; + } + } +} + static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node) { @@ -938,6 +989,7 @@ static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, } hlist_del_init(&node->tramp_hlist); tr->progs_cnt[kind]--; + bpf_trampoline_skip_prog(tr, node->link->prog); } static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, @@ -1246,11 +1298,9 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i]))) goto out; - /* This code will be executed even when the last bpf_tramp_image - * is alive. All progs are detached from the trampoline and the - * trampoline image is patched with jmp into epilogue to skip - * fexit progs. The fentry-only trampoline will be freed via - * multiple rcu callbacks. + /* + * All progs are detached and the last image has been freed, images + * hold a reference on the trampoline until then. */ hlist_del(&tr->hlist_key); hlist_del(&tr->hlist_ip); From d5288c519bbd0df9dff9f8dce2a8ce24e2ca9c8a Mon Sep 17 00:00:00 2001 From: "Florent Revest (Anthropic)" Date: Sat, 26 Sep 2026 13:56:00 +0000 Subject: [PATCH 05/11] selftests/bpf: Detach a trampoline prog while a task sleeps before it Add a test for the use-after-free fixed by the previous commit. A sleepable prog on bpf_fentry_test1() blocks a task on a userfaultfd page, like bpf_mod_race does, while the two progs that run after it in the same trampoline are detached and freed one after the other, which the test knows from their .bss maps going away. After the first detach the task is in an image that isn't the trampoline's current one anymore. The task is then released and must not call into the freed progs. This is done with fentry progs, with fexit progs, where the task is already past the original function, and with a task sleeping in a fentry prog while fexit progs are detached, where the original function must still be called. bpf_mod_race's userfaultfd helper moves to testing_helpers.c so that both tests can use it. Without the fix, on a kernel with KASAN: BUG: KASAN: vmalloc-out-of-bounds in __bpf_prog_enter_recur+0xed/0x1e0 Read of size 8 at addr ffa0000000144040 by task test_progs/171 CPU: 6 UID: 0 PID: 171 Comm: test_progs Tainted: G OE 7.2.0+ #1 PREEMPT(full) Call Trace: __bpf_prog_enter_recur+0xed/0x1e0 bpf_trampoline_6442545468+0x72/0xe3 bpf_fentry_test1+0x9/0x20 bpf_prog_test_run_tracing+0x183/0x3e0 __sys_bpf+0xd3f/0x38f0 ... Signed-off-by: Florent Revest (Anthropic) Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926135605.1217928-4-florent.revest@linux.dev --- .../selftests/bpf/prog_tests/bpf_mod_race.c | 34 +--- .../bpf/prog_tests/tramp_prog_detach.c | 188 ++++++++++++++++++ .../selftests/bpf/progs/tramp_prog_detach.c | 56 ++++++ tools/testing/selftests/bpf/testing_helpers.c | 28 +++ tools/testing/selftests/bpf/testing_helpers.h | 2 + 5 files changed, 275 insertions(+), 33 deletions(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/tramp_prog_detach.c create mode 100644 tools/testing/selftests/bpf/progs/tramp_prog_detach.c diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c b/tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c index ecc3d47919ad..f8497e764beb 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_mod_race.c @@ -55,38 +55,6 @@ static void *load_module_thread(void *p) return p; } -static int sys_userfaultfd(int flags) -{ - return syscall(__NR_userfaultfd, flags); -} - -static int test_setup_uffd(void *fault_addr) -{ - struct uffdio_register uffd_register = {}; - struct uffdio_api uffd_api = {}; - int uffd; - - uffd = sys_userfaultfd(O_CLOEXEC); - if (uffd < 0) - return -errno; - - uffd_api.api = UFFD_API; - uffd_api.features = 0; - if (ioctl(uffd, UFFDIO_API, &uffd_api)) { - close(uffd); - return -1; - } - - uffd_register.range.start = (unsigned long)fault_addr; - uffd_register.range.len = getpagesize(); - uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING; - if (ioctl(uffd, UFFDIO_REGISTER, &uffd_register)) { - close(uffd); - return -1; - } - return uffd; -} - static void test_bpf_mod_race_config(const struct test_config *config) { void *fault_addr, *skel_fail; @@ -117,7 +85,7 @@ static void test_bpf_mod_race_config(const struct test_config *config) if (!ASSERT_OK(bpf_mod_race__attach(skel), "bpf_mod_kfunc_race__attach")) goto end_destroy; - uffd = test_setup_uffd(fault_addr); + uffd = uffd_block_page(fault_addr); if (!ASSERT_GE(uffd, 0, "userfaultfd open + register address")) goto end_destroy; diff --git a/tools/testing/selftests/bpf/prog_tests/tramp_prog_detach.c b/tools/testing/selftests/bpf/prog_tests/tramp_prog_detach.c new file mode 100644 index 000000000000..1eb0d7237605 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/tramp_prog_detach.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "tramp_prog_detach.skel.h" +#include "testing_helpers.h" + +/* + * Detach and free progs while a task sleeps in the prog that runs before them + * in the same trampoline image, then let that task continue through the + * image. It must not call into the freed progs. + * + * The task is held in a sleepable prog with userfaultfd, like bpf_mod_race + * does. + */ + +static struct bpf_program *pick_prog(struct tramp_prog_detach *skel, + bool fexit, bool sleepable) +{ + if (fexit) + return sleepable ? skel->progs.fexit_sleepable : + skel->progs.fexit_victim; + return sleepable ? skel->progs.fentry_sleepable : + skel->progs.fentry_victim; +} + +static struct bpf_program *sleepable_prog; + +static void *run_sleepable(void *arg) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + + /* calls bpf_fentry_test1() */ + return (void *)(long)bpf_prog_test_run_opts(bpf_program__fd(sleepable_prog), + &topts); +} + +static struct tramp_prog_detach *load_one(bool fexit, bool sleepable) +{ + struct tramp_prog_detach *skel; + int err; + + skel = tramp_prog_detach__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return NULL; + + bpf_program__set_autoload(pick_prog(skel, fexit, sleepable), true); + err = tramp_prog_detach__load(skel); + if (!ASSERT_OK(err, "load")) + goto err; + skel->bss->pid = getpid(); + err = tramp_prog_detach__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto err; + return skel; +err: + tramp_prog_detach__destroy(skel); + return NULL; +} + +/* The .bss map of a destroyed skeleton goes away when its prog is freed */ +static bool wait_for_map_free(__u32 map_id) +{ + int i, fd; + + for (i = 0; i < 100; i++) { + fd = bpf_map_get_fd_by_id(map_id); + if (fd < 0) + return true; + close(fd); + usleep(100 * 1000); + } + return false; +} + +static void test_detach(bool sleepable_fexit, bool victim_fexit) +{ + struct tramp_prog_detach *sleepable = NULL, *victims[2] = {}; + struct pollfd pfd = { .events = POLLIN }; + struct uffdio_copy uffd_copy = {}; + struct bpf_map_info map_info = {}; + __u32 map_info_len = sizeof(map_info); + struct uffd_msg uffd_msg; + void *fault_page, *src_page = MAP_FAILED; + long page_size = getpagesize(); + bool started = false; + void *thread_ret; + pthread_t thread; + int i, uffd = -1; + + fault_page = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (!ASSERT_NEQ(fault_page, MAP_FAILED, "mmap fault_page")) + return; + src_page = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (!ASSERT_NEQ(src_page, MAP_FAILED, "mmap src_page")) + goto out; + + /* The most recently attached prog runs first */ + for (i = 0; i < ARRAY_SIZE(victims); i++) { + victims[i] = load_one(victim_fexit, false); + if (!victims[i]) + goto out; + } + sleepable = load_one(sleepable_fexit, true); + if (!sleepable) + goto out; + sleepable_prog = pick_prog(sleepable, sleepable_fexit, true); + + /* Not armed yet so this doesn't block, make sure sleepable runs first */ + if (!ASSERT_OK((long)run_sleepable(NULL), "dry run")) + goto out; + for (i = 0; i < ARRAY_SIZE(victims); i++) + if (!ASSERT_LT(sleepable->bss->ts, victims[i]->bss->ts, "prog order")) + goto out; + + uffd = uffd_block_page(fault_page); + if (!ASSERT_GE(uffd, 0, "userfaultfd open + register address")) + goto out; + sleepable->bss->fault_addr = fault_page; + + if (!ASSERT_OK(pthread_create(&thread, NULL, run_sleepable, NULL), + "pthread_create")) + goto out; + started = true; + + /* Wait for the thread to sleep in bpf_copy_from_user() */ + pfd.fd = uffd; + if (!ASSERT_EQ(poll(&pfd, 1, 10000), 1, "poll uffd")) + goto out; + if (!ASSERT_EQ(read(uffd, &uffd_msg, sizeof(uffd_msg)), sizeof(uffd_msg), + "read uffd")) + goto out; + if (!ASSERT_EQ(uffd_msg.event, UFFD_EVENT_PAGEFAULT, "uffd pagefault")) + goto out; + + /* + * Detach and unload the victim progs and wait for them to be freed. + * After the first one, the task is in an image that isn't the + * trampoline's current one anymore. + */ + for (i = 0; i < ARRAY_SIZE(victims); i++) { + if (!ASSERT_OK(bpf_map_get_info_by_fd(bpf_map__fd(victims[i]->maps.bss), + &map_info, &map_info_len), + "victim bss info")) + goto out; + tramp_prog_detach__destroy(victims[i]); + victims[i] = NULL; + if (!ASSERT_TRUE(wait_for_map_free(map_info.id), "victim freed")) + goto out; + } + +out: + /* Let the thread proceed with the rest of the trampoline */ + if (uffd >= 0) { + uffd_copy.dst = (unsigned long)fault_page; + uffd_copy.src = (unsigned long)src_page; + uffd_copy.len = page_size; + ASSERT_OK(ioctl(uffd, UFFDIO_COPY, &uffd_copy), "uffd copy"); + close(uffd); + } + if (started && + ASSERT_OK(pthread_join(thread, &thread_ret), "pthread_join")) + ASSERT_NULL(thread_ret, "blocking run"); + + for (i = 0; i < ARRAY_SIZE(victims); i++) + tramp_prog_detach__destroy(victims[i]); + tramp_prog_detach__destroy(sleepable); + if (src_page != MAP_FAILED) + munmap(src_page, page_size); + munmap(fault_page, page_size); +} + +void serial_test_tramp_prog_detach(void) +{ + /* a task sleeping before the original function is called */ + if (test__start_subtest("fentry")) + test_detach(false, false); + /* a task sleeping after the original function returned */ + if (test__start_subtest("fexit")) + test_detach(true, true); + /* the original function runs in between and must still be called */ + if (test__start_subtest("fentry_fexit")) + test_detach(false, true); +} diff --git a/tools/testing/selftests/bpf/progs/tramp_prog_detach.c b/tools/testing/selftests/bpf/progs/tramp_prog_detach.c new file mode 100644 index 000000000000..507d372167ec --- /dev/null +++ b/tools/testing/selftests/bpf/progs/tramp_prog_detach.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include +#include + +char _license[] SEC("license") = "GPL"; + +int pid; +void *fault_addr; +__u64 ts; + +static int do_sleepable(void) +{ + char dst; + + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + ts = bpf_ktime_get_ns(); + /* blocks for as long as user space wants when fault_addr is armed */ + bpf_copy_from_user(&dst, sizeof(dst), fault_addr); + return 0; +} + +static int do_victim(void) +{ + if (bpf_get_current_pid_tgid() >> 32 != pid) + return 0; + + ts = bpf_ktime_get_ns(); + return 0; +} + +SEC("?fentry.s/bpf_fentry_test1") +int BPF_PROG(fentry_sleepable, int a) +{ + return do_sleepable(); +} + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG(fentry_victim, int a) +{ + return do_victim(); +} + +SEC("?fexit.s/bpf_fentry_test1") +int BPF_PROG(fexit_sleepable, int a, int ret) +{ + return do_sleepable(); +} + +SEC("?fexit/bpf_fentry_test1") +int BPF_PROG(fexit_victim, int a, int ret) +{ + return do_victim(); +} diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index c970e7793dfc..9f672d0c9dda 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -13,6 +13,7 @@ #include "test_progs.h" #include "testing_helpers.h" #include +#include int parse_num_list(const char *s, bool **num_set, int *num_set_len) { @@ -459,6 +460,33 @@ int kern_sync_rcu(void) return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0); } +int uffd_block_page(void *fault_addr) +{ + struct uffdio_register uffd_register = {}; + struct uffdio_api uffd_api = {}; + int uffd; + + uffd = syscall(__NR_userfaultfd, O_CLOEXEC); + if (uffd < 0) + return -errno; + + uffd_api.api = UFFD_API; + uffd_api.features = 0; + if (ioctl(uffd, UFFDIO_API, &uffd_api)) { + close(uffd); + return -1; + } + + uffd_register.range.start = (unsigned long)fault_addr; + uffd_register.range.len = getpagesize(); + uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING; + if (ioctl(uffd, UFFDIO_REGISTER, &uffd_register)) { + close(uffd); + return -1; + } + return uffd; +} + int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt) { __u32 buf_element_size = sizeof(struct bpf_insn); diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 2edc6fb7fc52..1f83b6080716 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -36,6 +36,8 @@ __u64 read_perf_max_sample_freq(void); int load_bpf_testmod(bool verbose); int unload_bpf_testmod(bool verbose); int kern_sync_rcu(void); +/* returns a userfaultfd that makes accesses to fault_addr's page block */ +int uffd_block_page(void *fault_addr); int finit_module(int fd, const char *param_values, int flags); int delete_module(const char *name, int flags); int load_module(const char *path, bool verbose); From b88822d2584f78cbdad1d8256510dcac108b5630 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 1 Oct 2026 14:52:54 +0000 Subject: [PATCH 06/11] bpf: Fix packet range of pointers sharing an id Since commit 022ac0750883 ("bpf: use reg->var_off instead of reg->off for pointers"), find_good_pkt_pointers() sets the range of all packet pointers sharing an id from the umax of the compared pointer, and check_packet_access() requires umax + off + size <= range. That assumes the umax of two such pointers differ by exactly their constant distance. reg_bounds_sync() breaks it when var_off tightens one umax and not the other: r4 &= 0x38 if r4 > 50 goto exit ; umax 50, var_off (0x0; 0x38) r5 = pkt + r4 ; umax 50 r6 = r5 r6 += 8 ; umax 56, not 58 Comparing r6 with pkt_end sets the range to 56, and the valid 8-byte load at r5 is rejected (50 + 8 > 56). Comparing r5 sets it to 50, and the out-of-bounds 1-byte load at r6 - 7, i.e. r5 + 1, is accepted (56 - 7 + 1 <= 50). Don't call reg_bounds_sync() on a packet pointer that keeps its id (a constant was added or subtracted) or its range (an unknown non-negative value was subtracted), so that var_off cannot tighten its umax. Only update the 32-bit bounds from var_off: reg_bounds_sanity_check() wants them constant when the lower half of var_off is, e.g. for pkt + 8. This relies on nothing else changing the 64-bit bounds of a packet pointer, which holds today. var_off of such a pointer is no longer narrowed by its bounds. Adjust three verifier_align expectations; the low bits, which the alignment checks use, don't change. veristat on the selftests shows no verdict changes and +0.8% insns in test_cls_redirect_subprogs. Fixes: 022ac0750883 ("bpf: use reg->var_off instead of reg->off for pointers") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261001145255.855630-1-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 10 +++++++++- tools/testing/selftests/bpf/progs/verifier_align.c | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 41b49c56e123..5f874979b8d7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14826,7 +14826,15 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn "Tighten the scalar bounds before the arithmetic so the resulting pointer remains within the allowed range."); return -EINVAL; } - reg_bounds_sync(dst_reg); + /* + * A packet pointer that keeps its id or range is checked against a + * range set from the checked pointer's umax, so var_off must not tighten + * its umax. r32 must still match var_off for reg_bounds_sanity_check(). + */ + if (reg_is_pkt_pointer(dst_reg) && (known || dst_reg->range > 0)) + __update_reg32_bounds(dst_reg); + else + reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); if (bounds_ret == -EACCES) return bounds_ret; diff --git a/tools/testing/selftests/bpf/progs/verifier_align.c b/tools/testing/selftests/bpf/progs/verifier_align.c index 3e52686515ca..0083ef8b8ba7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_align.c +++ b/tools/testing/selftests/bpf/progs/verifier_align.c @@ -284,7 +284,7 @@ __msg("26: {{.*}} R5=pkt(r=8,imm=14)") */ __msg("28: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7fc){{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") /* Constant is added to R5 again, setting reg->off to 18. */ -__msg("29: {{.*}} R5=pkt(id=3,{{[^)]*}}var_off=(0x2; 0x7fc)") +__msg("29: {{.*}} R5=pkt(id=3,{{[^)]*}}var_off=(0x2; 0xffc)") /* And once more we add a variable; resulting {{[^)]*}}var_off * is still (4n), fixed offset is not changed. * Also, we create a new reg->id. @@ -359,7 +359,7 @@ __msg("7: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)") __msg("8: {{.*}} R6={{[^)]*}}var_off=(0x2; 0x7fc)") /* Packet pointer has (4n+2) offset */ __msg("11: {{.*}} R5={{[^)]*}}var_off=(0x2; 0x7fc)") -__msg("12: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x7fc)") +__msg("12: {{.*}} R4={{[^)]*}}var_off=(0x2; 0xffc)") /* At the time the word size load is performed from R5, * its total fixed offset is NET_IP_ALIGN + reg->off (0) * which is 2. Then the variable offset is (4n+2), so @@ -375,7 +375,7 @@ __msg("17: {{.*}} R6={{[^)]*}}var_off=(0x0; 0x3fc)") * another (4n+2). */ __msg("19: {{.*}} R5={{[^)]*}}var_off=(0x2; 0xffc)") -__msg("20: {{.*}} R4={{[^)]*}}var_off=(0x2; 0xffc)") +__msg("20: {{.*}} R4={{[^)]*}}var_off=(0x2; 0x1ffc)") /* At the time the word size load is performed from R5, * its total fixed offset is NET_IP_ALIGN + reg->off (0) * which is 2. Then the variable offset is (4n+2), so From 33a154a96e71a34a1bcca9f40da343dbbf7b38b4 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Thu, 1 Oct 2026 14:52:55 +0000 Subject: [PATCH 07/11] selftests/bpf: Test packet range of pointers sharing an id Add tests where two packet pointers share an id and tightening one pointer's umax from its var_off would put it less than their constant distance from the other's umax: with an index & 0x38 capped at 50, the base pointer keeps umax 50, so the pointer 8 bytes further on must keep umax 58, even though its known bits allow at most 56. These refused a valid program or accepted an out-of-bounds access before the fix: - check the advanced copy, load through the base: valid, was refused; - check the base, load the byte at base + 1 through a copy advanced by 8: was accepted; - check base + 4, load 4 bytes at base + 2 through base + 8: reads two bytes past the checked range, was accepted; - the same as the second with data_meta pointers checked against data: was accepted. These pass with and without the fix and cover nearby paths: - subtract an unknown scalar from a checked pointer and load below it (the range is kept across a new id); - reach a load through two paths whose checks cover 8 and 7 bytes after the loaded pointer; the second path must not be pruned by the first; - spill a copy of a pointer, check the pointer, fill the copy and load one byte past the checked range: the load is refused, and the copy has the range of the check. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20261001145255.855630-2-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/progs/verifier_direct_packet_access.c | 178 ++++++++++++++++++ .../bpf/progs/verifier_meta_access.c | 30 +++ 2 files changed, 208 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c index 915a9707298b..139ff019d87d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_direct_packet_access.c @@ -920,4 +920,182 @@ l1_%=: r0 = *(u8*)(r9 + 0); \ : __clobber_all); } +SEC("tc") +__description("direct packet access: 8-aligned offset, check p + 8, load 8 bytes at p") +__success __retval(0) __flag(BPF_F_ANY_ALIGNMENT) +__naked void pkt_same_id_check_copy_load_base(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0x38; \ + if r4 > 50 goto l0_%=; \ + /* r4 is a multiple of 8, at most 48 */ \ + r5 = r2; \ + r5 += r4; \ + r6 = r5; \ + r6 += 8; \ + if r6 > r3 goto l0_%=; \ + /* [r5, r5 + 8) is in the packet */ \ + r0 = *(u64*)(r5 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: 8-aligned offset, check p, load past it via p + 8") +__failure __msg("invalid access to packet, off=51 size=1") +__naked void pkt_same_id_check_base_load_via_copy(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0x38; \ + if r4 > 50 goto l0_%=; \ + /* r4 is a multiple of 8, at most 48 */ \ + r5 = r2; \ + r5 += r4; \ + r6 = r5; \ + r6 += 8; \ + if r5 > r3 goto l0_%=; \ + /* r6 - 7 is r5 + 1 */ \ + r0 = *(u8*)(r6 - 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: 8-aligned offset, check p + 4, 4-byte load at p + 2") +__failure __msg("invalid access to packet, off=52 size=4") +__naked void pkt_same_id_check_base_4_load_via_copy(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0x38; \ + if r4 > 50 goto l0_%=; \ + /* r4 is a multiple of 8, at most 48 */ \ + r5 = r2; \ + r5 += r4; \ + r6 = r5; \ + r6 += 8; \ + r7 = r5; \ + r7 += 4; \ + if r7 > r3 goto l0_%=; \ + /* [r5, r5 + 4) is in the packet, [r5 + 2, r5 + 6) may not be */ \ + r0 = *(u32*)(r6 - 6); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: checked pointer minus non-negative unknown keeps range") +__success __retval(0) __flag(BPF_F_ANY_ALIGNMENT) +__naked void pkt_sub_unknown_keeps_range(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0x1f; \ + r4 += 8; \ + r5 = r2; \ + r5 += 40; \ + if r5 > r3 goto l0_%=; \ + /* r5 is 8 to 39 bytes below the checked pointer */ \ + r5 -= r4; \ + r0 = *(u64*)(r5 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: no pruning of a path whose checks prove fewer bytes") +__failure __msg("invalid access to packet, off=255 size=8") +__flag(BPF_F_ANY_ALIGNMENT) __flag(BPF_F_TEST_STATE_FREQ) +__naked void pkt_same_id_pruning(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r0 = *(u32*)(r1 + %[__sk_buff_priority]); \ + r4 &= 0xff; \ + r5 = r2; \ + r5 += r4; \ + if r0 != 0 goto l1_%=; \ + /* this path proves [r5, r5 + 8) */ \ + r6 = r5; \ + r6 += 8; \ + if r6 > r3 goto l0_%=; \ + goto l2_%=; \ +l1_%=: /* this path proves [r5, r5 + 7) */ \ + if r5 > r3 goto l0_%=; \ + r6 = r5; \ + r6 += 7; \ + if r6 > r3 goto l0_%=; \ +l2_%=: r0 = *(u64*)(r5 + 0); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)), + __imm_const(__sk_buff_priority, offsetof(struct __sk_buff, priority)) + : __clobber_all); +} + +SEC("tc") +__description("direct packet access: spilled copy of checked pointer, load past range") +__failure __msg("invalid access to packet, off=262 size=1, R7(id={{[0-9]+}},off=262,r=262)") +__naked void pkt_spilled_copy_gets_range(void) +{ + asm volatile (" \ + r2 = *(u32*)(r1 + %[__sk_buff_data]); \ + r3 = *(u32*)(r1 + %[__sk_buff_data_end]); \ + r4 = *(u32*)(r1 + %[__sk_buff_mark]); \ + r4 &= 0xff; \ + r5 = r2; \ + r5 += r4; \ + *(u64*)(r10 - 8) = r5; \ + /* proves only bytes before r5 */ \ + if r5 > r3 goto l0_%=; \ + r6 = r5; \ + r6 += 7; \ + if r6 > r3 goto l0_%=; \ + /* [r5, r5 + 7) is in the packet */ \ + r7 = *(u64*)(r10 - 8); \ + r0 = *(u8*)(r7 + 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm_const(__sk_buff_data, offsetof(struct __sk_buff, data)), + __imm_const(__sk_buff_data_end, offsetof(struct __sk_buff, data_end)), + __imm_const(__sk_buff_mark, offsetof(struct __sk_buff, mark)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_meta_access.c b/tools/testing/selftests/bpf/progs/verifier_meta_access.c index 62235f032ffe..c87e0be4b2ff 100644 --- a/tools/testing/selftests/bpf/progs/verifier_meta_access.c +++ b/tools/testing/selftests/bpf/progs/verifier_meta_access.c @@ -281,4 +281,34 @@ l0_%=: r0 = 0; \ : __clobber_all); } +SEC("xdp") +__description("meta access, 8-aligned offset, check p, load a byte at p + 1 via p + 8") +__failure __msg("invalid access to packet, off=51 size=1") +__naked void meta_access_check_base_load_via_copy(void) +{ + asm volatile (" \ + r9 = r1; \ + call %[bpf_get_prandom_u32]; \ + r4 = r0; \ + r4 &= 0x38; \ + if r4 > 50 goto l0_%=; \ + /* r4 is a multiple of 8, at most 48 */ \ + r2 = *(u32*)(r9 + %[xdp_md_data_meta]); \ + r3 = *(u32*)(r9 + %[xdp_md_data]); \ + r5 = r2; \ + r5 += r4; \ + r6 = r5; \ + r6 += 8; \ + if r5 > r3 goto l0_%=; \ + /* r6 - 7 is r5 + 1 */ \ + r0 = *(u8*)(r6 - 7); \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm_const(xdp_md_data, offsetof(struct xdp_md, data)), + __imm_const(xdp_md_data_meta, offsetof(struct xdp_md, data_meta)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; From 47f4f695cec1eb82d961fdb466cd7c99d2865b7c Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 30 Sep 2026 09:59:18 +0000 Subject: [PATCH 08/11] bpf: Factor out __do_call_rcu_ttrace() Move the part of do_call_rcu_ttrace() that runs after call_rcu_ttrace_in_progress is set into __do_call_rcu_ttrace(). The next patch will call it from __free_rcu(). No functional change. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260930095920.601738-2-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/memalloc.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 8a8f088e83e6..08e4dde66cd5 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -298,19 +298,10 @@ static void enque_to_free(struct bpf_mem_cache *c, void *obj) llist_add(llnode, &c->free_by_rcu_ttrace); } -static void do_call_rcu_ttrace(struct bpf_mem_cache *c) +static void __do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode, *t; - if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { - if (unlikely(READ_ONCE(c->draining))) { - scoped_guard(raw_spinlock_irqsave, &c->lock) - llnode = llist_del_all(&c->free_by_rcu_ttrace); - free_all(c, llnode, !!c->percpu_size); - } - return; - } - WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace)) llist_add(llnode, &c->waiting_for_gp_ttrace); @@ -328,6 +319,22 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) call_rcu_tasks_trace(&c->rcu_ttrace, __free_rcu); } +static void do_call_rcu_ttrace(struct bpf_mem_cache *c) +{ + struct llist_node *llnode; + + if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { + if (unlikely(READ_ONCE(c->draining))) { + scoped_guard(raw_spinlock_irqsave, &c->lock) + llnode = llist_del_all(&c->free_by_rcu_ttrace); + free_all(c, llnode, !!c->percpu_size); + } + return; + } + + __do_call_rcu_ttrace(c); +} + static void free_bulk(struct bpf_mem_cache *c) { struct bpf_mem_cache *tgt = c->tgt; From fbd97dd1d3ce32d32fb7be86acc3fdb7c04fa043 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 30 Sep 2026 09:59:19 +0000 Subject: [PATCH 09/11] bpf: Fix objects stuck in free_by_rcu_ttrace do_call_rcu_ttrace() returns early when call_rcu_ttrace_in_progress is set and leaves the objects in free_by_rcu_ttrace. __free_rcu() frees waiting_for_gp_ttrace only and clears the flag. Hence the objects that free_bulk() or __free_by_rcu() added while RCU tasks trace GP was in flight stay in free_by_rcu_ttrace until free_bulk() or alloc_bulk() is called for the same bpf_mem_cache again, which may never happen. The number of such objects is not bounded. Turn call_rcu_ttrace_in_progress into three states: 0 - idle 1 - __free_rcu() is queued 2 - __free_rcu() is queued and free_by_rcu_ttrace got more objects since do_call_rcu_ttrace() sets 2. __free_rcu() does cmpxchg(1 -> 0) and starts the next GP when it fails. It cannot clear the flag first and check free_by_rcu_ttrace later, since bpf_mem_alloc_destroy() frees bpf_mem_cache without waiting for RCU callbacks when the flag is zero. Now __free_rcu() queues itself, so the one that didn't see 'draining' may do call_rcu_tasks_trace() after rcu_barrier_tasks_trace() in free_mem_alloc(). Queue it under rcu_read_lock() and do synchronize_rcu() before the barriers. Calling rcu_barrier_tasks_trace() twice works too, but creating and destroying hash maps in a loop on many cpus slows down to one free_mem_alloc() per GP and kworkers pile up. Fixes: 8d5a8011b35d ("bpf: Batch call_rcu callbacks instead of SLAB_TYPESAFE_BY_RCU.") Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260930095920.601738-3-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/memalloc.c | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 08e4dde66cd5..15684d0fc883 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -118,6 +118,11 @@ struct bpf_mem_cache { struct llist_head free_by_rcu_ttrace; struct llist_head waiting_for_gp_ttrace; struct rcu_head rcu_ttrace; + /* + * 0 - idle + * 1 - __free_rcu() is queued + * 2 - __free_rcu() is queued and free_by_rcu_ttrace got more objects since + */ atomic_t call_rcu_ttrace_in_progress; raw_spinlock_t lock; }; @@ -276,6 +281,8 @@ static int free_all(struct bpf_mem_cache *c, struct llist_node *llnode, bool per return cnt; } +static void __do_call_rcu_ttrace(struct bpf_mem_cache *c); + static void __free_rcu(struct rcu_head *head) { struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu_ttrace); @@ -285,7 +292,19 @@ static void __free_rcu(struct rcu_head *head) llnode = llist_del_all(&c->waiting_for_gp_ttrace); free_all(c, llnode, !!c->percpu_size); - atomic_set(&c->call_rcu_ttrace_in_progress, 0); + + /* + * do_call_rcu_ttrace() that ran while GP was in flight left its objects + * in free_by_rcu_ttrace. This cache may never free or alloc in bulk + * again, so start the next GP from here. + * 'c' can be freed as soon as call_rcu_ttrace_in_progress is zero. + */ + if (atomic_cmpxchg(&c->call_rcu_ttrace_in_progress, 1, 0) == 1) + return; + + /* Pairs with synchronize_rcu() in free_mem_alloc() */ + guard(rcu)(); + __do_call_rcu_ttrace(c); } static void enque_to_free(struct bpf_mem_cache *c, void *obj) @@ -302,6 +321,12 @@ static void __do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode, *t; + /* + * Must be done before llist_del_all(). Objects that it misses were + * added by do_call_rcu_ttrace() that will set 2 after this store. + */ + atomic_set(&c->call_rcu_ttrace_in_progress, 1); + WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp_ttrace)); llist_for_each_safe(llnode, t, llist_del_all(&c->free_by_rcu_ttrace)) llist_add(llnode, &c->waiting_for_gp_ttrace); @@ -323,7 +348,7 @@ static void do_call_rcu_ttrace(struct bpf_mem_cache *c) { struct llist_node *llnode; - if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 1)) { + if (atomic_xchg(&c->call_rcu_ttrace_in_progress, 2)) { if (unlikely(READ_ONCE(c->draining))) { scoped_guard(raw_spinlock_irqsave, &c->lock) llnode = llist_del_all(&c->free_by_rcu_ttrace); @@ -707,7 +732,12 @@ static void free_mem_alloc(struct bpf_mem_alloc *ma) * to wait for the pending __free_by_rcu(), and __free_rcu(). RCU Tasks * Trace grace period implies RCU grace period, so all __free_rcu don't * need extra call_rcu() (and thus extra rcu_barrier() here). + * + * __free_rcu() queues itself again unless it sees 'draining'. After + * synchronize_rcu() it either did that already or will not do it, so + * rcu_barrier_tasks_trace() cannot miss it. */ + synchronize_rcu(); rcu_barrier(); /* wait for __free_by_rcu */ rcu_barrier_tasks_trace(); /* wait for __free_rcu */ free_mem_alloc_no_barrier(ma); From aeb709c767aeca996e6011133e4f7bdb2a4af652 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 30 Sep 2026 09:59:20 +0000 Subject: [PATCH 10/11] selftests/bpf: Add a test for objects stuck in free_by_rcu_ttrace Delete all elements of BPF_F_NO_PREALLOC hash map in one batch. The first free_bulk() starts RCU tasks trace GP and the rest of the elements are freed while it's in flight. Wait for call_rcu_ttrace_in_progress to clear in bpf_mem_cache of every cpu and check that free_by_rcu_ttrace and waiting_for_gp_ttrace lists are empty. Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260930095920.601738-4-alexei.starovoitov@gmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/prog_tests/bpf_ma_ttrace.c | 60 +++++++++++++++++++ .../selftests/bpf/progs/bpf_ma_ttrace.c | 50 ++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/bpf_ma_ttrace.c create mode 100644 tools/testing/selftests/bpf/progs/bpf_ma_ttrace.c diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_ma_ttrace.c b/tools/testing/selftests/bpf/prog_tests/bpf_ma_ttrace.c new file mode 100644 index 000000000000..a1d41b109940 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpf_ma_ttrace.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "bpf_ma_ttrace.skel.h" + +#define NR_ELEMS 4096 + +/* + * The first free_bulk() starts RCU tasks trace GP. The rest of the elements are + * deleted while it's in flight. They should be freed without further alloc or + * free from this map. + */ +void test_bpf_ma_ttrace(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct bpf_ma_ttrace *skel; + __u32 cnt = NR_ELEMS; + long *vals = NULL; + int *keys = NULL; + int i, err, fd, nr_cpus; + + skel = bpf_ma_ttrace__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + nr_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(nr_cpus, 0, "nr_cpus")) + goto out; + skel->bss->nr_cpus = nr_cpus; + + keys = calloc(NR_ELEMS, sizeof(*keys)); + vals = calloc(NR_ELEMS, sizeof(*vals)); + if (!ASSERT_OK_PTR(keys, "keys") || !ASSERT_OK_PTR(vals, "vals")) + goto out; + for (i = 0; i < NR_ELEMS; i++) + keys[i] = i; + + fd = bpf_map__fd(skel->maps.htab); + err = bpf_map_update_batch(fd, keys, vals, &cnt, NULL); + if (!ASSERT_OK(err, "update_batch") || !ASSERT_EQ(cnt, NR_ELEMS, "update_cnt")) + goto out; + err = bpf_map_delete_batch(fd, keys, &cnt, NULL); + if (!ASSERT_OK(err, "delete_batch") || !ASSERT_EQ(cnt, NR_ELEMS, "delete_cnt")) + goto out; + + /* Wait for all __free_rcu() callbacks to finish */ + for (i = 0; i < 300; i++) { + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.check_ttrace), &opts); + if (!ASSERT_OK(err, "test_run") || !ASSERT_OK(opts.retval, "retval")) + goto out; + if (!skel->bss->in_progress) + break; + usleep(100000); + } + ASSERT_EQ(skel->bss->nr_caches, nr_cpus, "nr_caches"); + ASSERT_EQ(skel->bss->in_progress, 0, "in_progress"); + ASSERT_EQ(skel->bss->not_freed, 0, "not_freed"); +out: + free(keys); + free(vals); + bpf_ma_ttrace__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/bpf_ma_ttrace.c b/tools/testing/selftests/bpf/progs/bpf_ma_ttrace.c new file mode 100644 index 000000000000..31b2a962e339 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/bpf_ma_ttrace.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "bpf_kfuncs.h" + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 4096); + __type(key, int); + __type(value, long); +} htab SEC(".maps"); + +extern const void __per_cpu_offset __ksym; + +int nr_cpus; +int nr_caches; +int in_progress; +int not_freed; + +/* Look at bpf_mem_cache of every cpu that htab allocates its elements from */ +SEC("syscall") +int check_ttrace(void *ctx) +{ + struct bpf_htab *h = bpf_core_cast(&htab, struct bpf_htab); + unsigned long cache = (unsigned long)BPF_CORE_READ(h, ma.cache); + const unsigned long *offsets = &__per_cpu_offset; + struct bpf_mem_cache *c; + unsigned long off; + int cpu; + + nr_caches = 0; + in_progress = 0; + not_freed = 0; + bpf_for(cpu, 0, nr_cpus) { + if (bpf_probe_read_kernel(&off, sizeof(off), offsets + cpu)) + return -1; + c = bpf_core_cast((void *)(cache + off), struct bpf_mem_cache); + if (c->unit_size) + nr_caches++; + if (c->call_rcu_ttrace_in_progress.counter) + in_progress++; + if (c->free_by_rcu_ttrace.first || c->waiting_for_gp_ttrace.first) + not_freed++; + } + return 0; +} + +char _license[] SEC("license") = "GPL"; From de020dc8049bfb2b22e3b6d99c031feb2e22d112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Mete=20Kaya?= Date: Tue, 29 Sep 2026 11:10:34 +0300 Subject: [PATCH 11/11] bpf: Fix missing migration protection in __rhtab_map_lookup_and_delete_batch() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bpf_mem_cache_free_rcu() uses this_cpu_ptr() which requires migration to be disabled. All callers of rhtab_delete_elem() disable migration except __rhtab_map_lookup_and_delete_batch(), which calls it under rcu_read_lock() only. On CONFIG_PREEMPT_RCU, rcu_read_lock() does not disable preemption or migration, so the task can migrate between CPUs during the delete loop, causing this_cpu_ptr() to trigger: BUG: using smp_processor_id() in preemptible [00000000] code Fix by wrapping the delete loop in migrate_disable()/migrate_enable() in __rhtab_map_lookup_and_delete_batch(), matching the migration protection that the other callers already provide. Fixes: 818e00848227 ("bpf: Implement iteration ops for resizable hashtab") Reported-by: syzbot+fd7e415d891073b83e1f@syzkaller.appspotmail.com Signed-off-by: Ömer Mete Kaya Signed-off-by: Alexei Starovoitov Acked-by: Mykyta Yatsenko Link: https://patch.msgid.link/20260929081609.557899-1-omermetekaya0@gmail.com Closes: https://syzkaller.appspot.com/bug?extid=fd7e415d891073b83e1f --- kernel/bpf/hashtab.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 548eb1cc7c45..13a2356c84cf 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -3374,8 +3374,10 @@ static int __rhtab_map_lookup_and_delete_batch(struct bpf_map *map, } if (do_delete) { + migrate_disable(); for (i = 0; i < total; i++) rhtab_delete_elem(rhtab, del_elems[i], NULL, 0); + migrate_enable(); } rcu_read_unlock();