From 232475c32994411bd816d6358ca48e94010fc84a Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 22 Oct 2026 16:48:36 +0800 Subject: [PATCH 01/48] bpf: Protect the struct_ops association with a spinlock bpf_prog_assoc_struct_ops() and bpf_prog_disassoc_struct_ops() update prog->aux->st_ops_assoc under st_ops_assoc_mutex, and the latter puts the map of a non-struct_ops prog while holding it. A struct_ops map is about to dissociate its progs from map_free_pre_rcu(), which bpf_map_put() calls in whatever context drops the last reference, including softirq, e.g. bpf_struct_ops_put() from tcp_cleanup_congestion_control(). A mutex can't be taken there, and the put of the last reference under the lock of one prog would take the lock of other progs. Replace the mutex with a spinlock taken with interrupts disabled, and put the map after dropping it. Signed-off-by: Qiliang Yuan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261022-selftests-bpf-struct-ops-assoc-timer-v6-1-2c0f18e82582@gmail.com --- include/linux/bpf.h | 2 +- kernel/bpf/bpf_struct_ops.c | 19 ++++++++++--------- kernel/bpf/core.c | 3 +-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 580f89c450b5..17b2a425e05e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1979,7 +1979,7 @@ struct bpf_prog_aux { struct rcu_work rwork; }; struct bpf_stream *stream[2]; - struct mutex st_ops_assoc_mutex; + spinlock_t st_ops_assoc_lock; /* protects st_ops_assoc */ struct bpf_map __rcu *st_ops_assoc; }; diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 1178acd72296..a1ae30b145a0 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1541,10 +1541,10 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map) { struct bpf_map *st_ops_assoc; - guard(mutex)(&prog->aux->st_ops_assoc_mutex); + guard(spinlock_irqsave)(&prog->aux->st_ops_assoc_lock); st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc, - lockdep_is_held(&prog->aux->st_ops_assoc_mutex)); + lockdep_is_held(&prog->aux->st_ops_assoc_lock)); if (st_ops_assoc && st_ops_assoc == map) return 0; @@ -1569,19 +1569,20 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map) void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog) { + struct bpf_prog_aux *aux = prog->aux; struct bpf_map *st_ops_assoc; - guard(mutex)(&prog->aux->st_ops_assoc_mutex); + scoped_guard(spinlock_irqsave, &aux->st_ops_assoc_lock) { + st_ops_assoc = rcu_dereference_protected(aux->st_ops_assoc, + lockdep_is_held(&aux->st_ops_assoc_lock)); + if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON) + return; - st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc, - lockdep_is_held(&prog->aux->st_ops_assoc_mutex)); - if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON) - return; + RCU_INIT_POINTER(aux->st_ops_assoc, NULL); + } if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) bpf_map_put(st_ops_assoc); - - RCU_INIT_POINTER(prog->aux->st_ops_assoc, NULL); } /* diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 3030beb531d9..0a8ba603f44b 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -141,7 +141,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag mutex_init(&fp->aux->used_maps_mutex); mutex_init(&fp->aux->ext_mutex); mutex_init(&fp->aux->dst_mutex); - mutex_init(&fp->aux->st_ops_assoc_mutex); + spin_lock_init(&fp->aux->st_ops_assoc_lock); return fp; } @@ -291,7 +291,6 @@ void __bpf_prog_free(struct bpf_prog *fp) #endif mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); - mutex_destroy(&fp->aux->st_ops_assoc_mutex); kfree(fp->aux->poke_tab); kfree(fp->aux); } From 377ab5f2ef793b034d7d710212604fd62f0f00b9 Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 22 Oct 2026 16:48:37 +0800 Subject: [PATCH 02/48] bpf: Dissociate struct_ops progs before the grace period of the map A struct_ops prog is associated with its map without a reference on the map, and finds the struct_ops through bpf_prog_get_assoc_struct_ops() under RCU. For a struct_ops map, bpf_map_put() waits for an RCU grace period before it calls bpf_struct_ops_map_free(), which clears the association and then frees the trampoline and the kdata right away. Nothing waits for a prog that reads the association after that grace period started and before the association is cleared. A timer callback of a struct_ops prog that calls test_1 of bpf_testmod's multi_st_ops through the association while the map is freed returns into the freed trampoline, whose image is filled with int3. RAX holds MAP_MAGIC (1234), which test_1 has just returned: Oops: int3: 0000 [#1] SMP KASAN NOPTI CPU: 2 UID: 0 PID: 31 Comm: ksoftirqd/2 ... RIP: 0010:0xffffffffc043114d Code: cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc cc RAX: 00000000000004d2 ... Call Trace: bpf_prog_ec591f4f9d13f364_timer_cb+0x96/0x1e2 bpf_timer_cb+0x15d/0x240 __hrtimer_run_queues+0x264/0x5a0 hrtimer_run_softirq+0x1a7/0x3b0 handle_softirqs+0x18c/0x5b0 Dissociate the progs in map_free_pre_rcu(), so that the grace period of bpf_map_put() follows it, and make that an RCU tasks trace grace period for every struct_ops map. It then also waits for the progs that read the association under RCU tasks trace, e.g. from a bpf_wq callback or a sleepable prog, as a tasks trace grace period implies an RCU one. Stop setting free_after_rcu_gp, which bpf_map_put() no longer reads for a struct_ops map, and update the comment of free_after_mult_rcu_gp in struct bpf_struct_ops, which now only picks the grace period of a cgroup prog array. The st_ops_assoc_in_timer_free subtest of test_progs reproduces it. Its timer callback keeps calling test_1 through the association while the map is freed, and test_1 spins in bpf_loop() when called from there: $ cd tools/testing/selftests/bpf $ make test_progs $ ./test_progs -t struct_ops_assoc/st_ops_assoc_in_timer_free On kernels of the same bpf-next commit with KASAN enabled, x86_64 VM with 32 vCPUs, before and after this patch: before after outcome Oops (int3) on run 1 100 of 100 runs passed, no Oops Fixes: 5db69b0fbdc8 ("bpf: Make struct_ops tasks_rcu grace period optional") Signed-off-by: Qiliang Yuan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261022-selftests-bpf-struct-ops-assoc-timer-v6-2-2c0f18e82582@gmail.com --- include/linux/bpf.h | 8 +++++--- kernel/bpf/bpf_struct_ops.c | 10 ++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 17b2a425e05e..ff0436781293 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2240,9 +2240,11 @@ struct btf_member; * and its progs. It is unnecessary if the @unreg * has waited for the correct rcu gp or the @unreg * has ensured all struct_ops prog has finished running. - * @free_after_mult_rcu_gp: Same as @free_after_tasks_rcu_gp but waiting for - * both tasks_trace_rcu and regular rcu grace period. - * It is usually needed if the struct_ops has sleepable prog. + * @free_after_mult_rcu_gp: Set to true if the cgroup prog array of this + * struct_ops needs a tasks_trace_rcu gp, besides a + * regular rcu gp, before it is freed. It is usually + * needed if the struct_ops has sleepable prog. A + * struct_ops map always waits for both. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index a1ae30b145a0..7c5cd6086e27 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1030,6 +1030,7 @@ static void bpf_struct_ops_map_free_pre_rcu(struct bpf_map *map) struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; bpf_struct_ops_map_del_ksyms(st_map); + bpf_struct_ops_map_dissoc_progs(st_map); } static void bpf_struct_ops_map_free(struct bpf_map *map) @@ -1045,8 +1046,6 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) if (btf_is_module(st_map->btf)) module_put(st_map->st_ops_desc->st_ops->owner); - bpf_struct_ops_map_dissoc_progs(st_map); - if (tasks_rcu && IS_ENABLED(CONFIG_TASKS_RCU)) synchronize_rcu_tasks(); @@ -1162,8 +1161,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) mutex_init(&st_map->lock); bpf_map_init_from_attr(map, attr); - map->free_after_mult_rcu_gp = st_ops_desc->st_ops->free_after_mult_rcu_gp; - map->free_after_rcu_gp = true; + /* + * A prog dissociated in map_free_pre_rcu() may still run the struct_ops + * from a bpf_wq callback or a sleepable prog, under RCU tasks trace. + */ + map->free_after_mult_rcu_gp = true; return map; From a451867eb5f645a199d0f70cb2b1f541e07f33ec Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 22 Oct 2026 16:48:38 +0800 Subject: [PATCH 03/48] selftests/bpf: Test freeing a struct_ops map under a running prog st_ops_assoc_in_timer_no_uref drops every reference to the struct_ops map while a timer callback of its prog calls .test_1 through the association, and checks that the callback ends up with no struct_ops. The timer first fires 500ms later and .test_1 returns right away, so the callback hardly ever runs .test_1 while the map is freed, and the test can't catch a map that is freed under a running .test_1. Add st_ops_assoc_in_timer_free, which starts the timer right away, restarts it with no delay until the association is gone, and makes .test_1 spin in bpf_loop() when the timer callback calls it. Share the steps of st_ops_assoc_in_timer_no_uref with it through a helper, which stops when associating or running syscall_prog fails instead of waiting for a timer that never starts. st_ops_assoc_in_timer_free run in a loop on kernels of the same bpf-next commit with KASAN enabled, x86_64 VM with 32 vCPUs, without and with the previous fix: without the fix with the fix outcome Oops (int3) on run 1 100 of 100 passed, median 0.28 s Signed-off-by: Qiliang Yuan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261022-selftests-bpf-struct-ops-assoc-timer-v6-3-2c0f18e82582@gmail.com --- .../bpf/prog_tests/test_struct_ops_assoc.c | 37 ++++++++++++++++--- .../bpf/progs/struct_ops_assoc_in_timer.c | 16 ++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c index 461ded722351..6c85d368bf16 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c @@ -136,7 +136,8 @@ static void test_st_ops_assoc_in_timer(void) struct_ops_assoc_in_timer__destroy(skel); } -static void test_st_ops_assoc_in_timer_no_uref(void) +static void run_st_ops_assoc_in_timer_no_uref(int timer_ns, int wait_map_free, + int spin_loops) { struct struct_ops_assoc_in_timer *skel = NULL; struct bpf_link *link; @@ -148,7 +149,8 @@ static void test_st_ops_assoc_in_timer_no_uref(void) err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog, skel->maps.st_ops_map, NULL); - ASSERT_OK(err, "bpf_program__assoc_struct_ops"); + if (!ASSERT_OK(err, "bpf_program__assoc_struct_ops")) + goto out; link = bpf_map__attach_struct_ops(skel->maps.st_ops_map); if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) @@ -157,12 +159,18 @@ static void test_st_ops_assoc_in_timer_no_uref(void) /* * Run .test_1 by calling kfunc bpf_kfunc_multi_st_ops_test_1_prog_arg() and checks * the return value. .test_1 will also schedule timer_cb that runs .test_1 again. - * timer_cb will run 500ms after syscall_prog runs, when the user space no longer - * holds a reference to st_ops_map. + * timer_cb will run timer_ns after syscall_prog runs and, with wait_map_free, every + * timer_ns after that until the map is freed. With 500ms, it first runs when the + * user space no longer holds a reference to st_ops_map. */ - skel->bss->timer_ns = 500000000; + skel->bss->timer_ns = timer_ns; + skel->bss->wait_map_free = wait_map_free; + skel->bss->spin_loops = spin_loops; err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog), NULL); - ASSERT_OK(err, "bpf_prog_test_run_opts"); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) { + bpf_link__destroy(link); + goto out; + } /* Detach and close struct_ops map to cause it to be freed */ bpf_link__destroy(link); @@ -178,6 +186,21 @@ static void test_st_ops_assoc_in_timer_no_uref(void) struct_ops_assoc_in_timer__destroy(skel); } +static void test_st_ops_assoc_in_timer_no_uref(void) +{ + run_st_ops_assoc_in_timer_no_uref(500000000, 0, 0); +} + +/* + * Keep calling .test_1 through the association from the timer callback, and + * stay in it for a while each time, while the map is being freed. The map + * must not be freed under a running .test_1. + */ +static void test_st_ops_assoc_in_timer_free(void) +{ + run_st_ops_assoc_in_timer_no_uref(0, 1, 1 << 20); +} + void test_struct_ops_assoc(void) { if (test__start_subtest("st_ops_assoc")) @@ -188,4 +211,6 @@ void test_struct_ops_assoc(void) test_st_ops_assoc_in_timer(); if (test__start_subtest("st_ops_assoc_in_timer_no_uref")) test_st_ops_assoc_in_timer_no_uref(); + if (test__start_subtest("st_ops_assoc_in_timer_free")) + test_st_ops_assoc_in_timer_free(); } diff --git a/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c b/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c index 0bed49e9f217..47a44c8e8b11 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c @@ -25,6 +25,13 @@ int test_err; int timer_ns; int timer_test_1_ret; int timer_cb_run; +int wait_map_free; +int spin_loops; + +static int spin(u32 i, void *ctx) +{ + return 0; +} __noinline static int timer_cb(void *map, int *key, struct bpf_timer *timer) { @@ -34,6 +41,12 @@ __noinline static int timer_cb(void *map, int *key, struct bpf_timer *timer) timer_test_1_ret = bpf_kfunc_multi_st_ops_test_1_assoc(&args); recur--; + /* Try again later until the map is freed */ + if (wait_map_free && timer_test_1_ret != -1) { + bpf_timer_start(timer, timer_ns, 0); + return 0; + } + timer_cb_run++; return 0; @@ -53,6 +66,9 @@ int BPF_PROG(test_1, struct st_ops_args *args) bpf_timer_init(timer, &array_map, 1); bpf_timer_set_callback(timer, timer_cb); bpf_timer_start(timer, timer_ns, 0); + } else if (spin_loops) { + /* Stay in the trampoline while the map may be freed */ + bpf_loop(spin_loops, spin, NULL, 0); } return MAP_MAGIC; From ac3df711a428a9ff4277100ee26c6765e8f80a3b Mon Sep 17 00:00:00 2001 From: Qiliang Yuan Date: Thu, 22 Oct 2026 16:48:39 +0800 Subject: [PATCH 04/48] selftests/bpf: Wait for the struct_ops map to be freed in assoc_in_timer st_ops_assoc_in_timer_no_uref runs .test_1, which starts a timer to call it again 500ms later, drops every reference to the struct_ops map, and expects the timer callback to find no struct_ops associated with its program anymore. The association goes away only when the last reference to the map is dropped. syscall_prog holds one through its own association and drops it only when it is freed, in a work item after an RCU tasks trace grace period. With other tests running in parallel, that can take longer than 500ms, and the callback still reaches the struct_ops and gets MAP_MAGIC instead of -1. Have the timer callback restart the timer until the map is gone, as it does for st_ops_assoc_in_timer_free. Calling the struct_ops through the association until then is safe, as the map is freed only after a grace period that follows clearing the association. st_ops_assoc_in_timer_no_uref run 100 times with test_progs -j32 running in the background, on a KASAN kernel with the fix, x86_64 VM with 32 vCPUs, before and after this patch: before after outcome 2 of 100 failed (MAP_MAGIC) 0 of 100 failed (MAP_MAGIC) Fixes: 0e841d19263a ("selftests/bpf: Test getting associated struct_ops in timer callback") Signed-off-by: Qiliang Yuan Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261022-selftests-bpf-struct-ops-assoc-timer-v6-4-2c0f18e82582@gmail.com --- tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c index 6c85d368bf16..9b646d134a45 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c @@ -188,7 +188,7 @@ static void run_st_ops_assoc_in_timer_no_uref(int timer_ns, int wait_map_free, static void test_st_ops_assoc_in_timer_no_uref(void) { - run_st_ops_assoc_in_timer_no_uref(500000000, 0, 0); + run_st_ops_assoc_in_timer_no_uref(500000000, 1, 0); } /* From 02da62c2f89668fa1adafa3cc806025b560e45ce Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Wed, 21 Oct 2026 22:08:44 +0000 Subject: [PATCH 05/48] bpf, x86: Use emit_rsb_call() to call bpf_unwind_target() clang inlines emit_unwind_ret_addr() and emit_patch() into do_jit() and folds 'bpf_unwind_target - X86_PATCH_SIZE' into one relocation. It points to a nop in the padding in front of the function, and objtool complains: vmlinux.o: warning: objtool: do_jit+0x1330: relocation to !ENDBR: .text+0xc7fab It's only used to compute rel32 of a direct call, so nothing is wrong at run time. emit_rsb_call() does call depth accounting too and hides func from the optimizer. Use it instead of open coding. Fixes: cff94a3d78d8 ("bpf, x86: Support unwind_goto and unwind_exit") Reported-by: Nathan Chancellor Signed-off-by: Alexei Starovoitov Tested-by: Nathan Chancellor # build Link: https://patch.msgid.link/20261021220844.239400-1-alexei.starovoitov@gmail.com Closes: https://lore.kernel.org/bpf/20261021201456.GA3197963@ax162/ --- arch/x86/net/bpf_jit_comp.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 6a2fe3d0039d..71f7200dba00 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2564,9 +2564,7 @@ static int emit_unwind_ret_addr(u8 **pprog, u8 *ip) EMIT2(0x50, 0x56); /* push rax; push rsi */ EMIT4(0x48, 0x8B, 0x7D, 0x08); /* mov rdi, [rbp + 8] */ - ip += prog - start; - ip += x86_call_depth_emit_accounting(&prog, bpf_unwind_target, ip); - if (emit_call(&prog, bpf_unwind_target, ip)) + if (emit_rsb_call(&prog, bpf_unwind_target, ip + (prog - start))) return -EINVAL; EMIT4(0x48, 0x89, 0x45, 0x08); /* mov [rbp + 8], rax */ EMIT2(0x5E, 0x58); /* pop rsi; pop rax */ From fb2b27d214c3b88d926a303267dd83cfdb428b13 Mon Sep 17 00:00:00 2001 From: Feng Yang Date: Thu, 22 Oct 2026 14:47:27 +0800 Subject: [PATCH 06/48] bpf: Propagate btf_add_type() errors in btf_check_all_metas() btf_check_all_metas() ignores the return value of btf_add_type(), so a type that could not be registered is silently dropped while the loop keeps walking over the type section. Both failure modes are reachable: - -E2BIG. Once the types array reaches its capacity (BTF_MAX_TYPE), every following btf_add_type() fails the same way, so btf_check_all_metas() still returns 0 and the BTF object is created with btf->nr_types capped while hdr->type_len claims more records. - -ENOMEM. btf->nr_types is not incremented and the array is not grown, so the next type is recorded at the slot of the dropped one and btf->types[] stays off by one with respect to the type ids for all remaining types. btf_type_by_id() then returns the wrong type for those ids for the whole lifetime of the object, and the last type is silently left unresolved. Propagate the error so that the object is rejected instead. Fixes: 69b693f0aefa ("bpf: btf: Introduce BPF Type Format (BTF)") Signed-off-by: Feng Yang Signed-off-by: Alexei Starovoitov Reviewed-by: Mahe Tardy Acked-by: Alan Maguire Link: https://patch.msgid.link/20261022064727.747304-1-yangfeng59949@163.com --- kernel/bpf/btf.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 538129ed2418..94d43612d211 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -5871,6 +5871,7 @@ static int btf_check_all_metas(struct btf_verifier_env *env) struct btf *btf = env->btf; struct btf_header *hdr; void *cur, *end; + int err; hdr = &btf->hdr; cur = btf->nohdr_data + hdr->type_off; @@ -5885,7 +5886,10 @@ static int btf_check_all_metas(struct btf_verifier_env *env) if (meta_size < 0) return meta_size; - btf_add_type(env, t); + err = btf_add_type(env, t); + if (err) + return err; + cur += meta_size; env->log_type_id++; } From ebf0ab370bcdcd17f4faa204e1d2cc8e817ca6b1 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Thu, 22 Oct 2026 05:04:55 -0700 Subject: [PATCH 07/48] bpf: Size backward jumps by the insns they jump over A backward jump is sized from its target's current-pass address and its own previous-pass address, so it can grow while other jumps shrink. Two passes may then give the same image length with jumps still changing, which ends in "bpf_jit: fatal error", and a jump growing in a padding pass ends in "unexpected jmp_cond padding". Shift the jump's previous-pass address by how far the current pass has moved it, so a backward jump is sized by the insns it jumps over, like a forward one. Also start every insn at BPF_MAX_INSN_SIZE instead of 64, so none can grow in the first pass. Then no insn grows from one pass to the next, and a repeated image length means nothing moved. The 123 limit on imm8 offsets worked around that growth and is dropped. The ja -1 special case is subsumed, such a jump is now imm8 from the first pass. Suggested-by: Alexei Starovoitov Signed-off-by: Mykyta Yatsenko Signed-off-by: Alexei Starovoitov Acked-by: Daniel Borkmann Link: https://patch.msgid.link/20261022-jit_size_fix-v2-1-4bee6921b218@meta.com --- arch/x86/net/bpf_jit_comp.c | 109 ++++++++++-------------------------- 1 file changed, 31 insertions(+), 78 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 71f7200dba00..fd65ac84cfc7 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -97,56 +97,6 @@ static bool is_imm8(int value) return value <= 127 && value >= -128; } -/* - * Let us limit the positive offset to be <= 123. - * This is to ensure eventual jit convergence For the following patterns: - * ... - * pass4, final_proglen=4391: - * ... - * 20e: 48 85 ff test rdi,rdi - * 211: 74 7d je 0x290 - * 213: 48 8b 77 00 mov rsi,QWORD PTR [rdi+0x0] - * ... - * 289: 48 85 ff test rdi,rdi - * 28c: 74 17 je 0x2a5 - * 28e: e9 7f ff ff ff jmp 0x212 - * 293: bf 03 00 00 00 mov edi,0x3 - * Note that insn at 0x211 is 2-byte cond jump insn for offset 0x7d (-125) - * and insn at 0x28e is 5-byte jmp insn with offset -129. - * - * pass5, final_proglen=4392: - * ... - * 20e: 48 85 ff test rdi,rdi - * 211: 0f 84 80 00 00 00 je 0x297 - * 217: 48 8b 77 00 mov rsi,QWORD PTR [rdi+0x0] - * ... - * 28d: 48 85 ff test rdi,rdi - * 290: 74 1a je 0x2ac - * 292: eb 84 jmp 0x218 - * 294: bf 03 00 00 00 mov edi,0x3 - * Note that insn at 0x211 is 6-byte cond jump insn now since its offset - * becomes 0x80 based on previous round (0x293 - 0x213 = 0x80). - * At the same time, insn at 0x292 is a 2-byte insn since its offset is - * -124. - * - * pass6 will repeat the same code as in pass4 and this will prevent - * eventual convergence. - * - * To fix this issue, we need to break je (2->6 bytes) <-> jmp (5->2 bytes) - * cycle in the above. In the above example je offset <= 0x7c should work. - * - * For other cases, je <-> je needs offset <= 0x7b to avoid no convergence - * issue. For jmp <-> je and jmp <-> jmp cases, jmp offset <= 0x7c should - * avoid no convergence issue. - * - * Overall, let us limit the positive offset for 8bit cond/uncond jmp insn - * to maximum 123 (0x7b). This way, the jit pass can eventually converge. - */ -static bool is_imm8_jmp_offset(int value) -{ - return value <= 123 && value >= -128; -} - static bool is_simm32(s64 value) { return value == (s64)(s32)value; @@ -2298,6 +2248,17 @@ static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr) #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp))) +/* + * addrs[] below i are from this pass, addrs[i] and above are from the previous + * one. 'moved' is how far back this pass moved the start of the current insn. + * With it a jump in either direction is sized by the insns it jumps over, plus + * its own previous size for a backward jump, none of which grow. + */ +static int jmp_distance(const int *addrs, int i, int off, int moved) +{ + return addrs[i + off] - addrs[i] + (off < 0 ? moved : 0); +} + #define __LOAD_TCC_PTR(off) \ emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, off) /* mov rax, qword ptr [rbp - rounded_stack_depth - 16] */ @@ -2586,7 +2547,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * u64 arena_vm_start = 0, user_vm_start = 0; void __percpu *priv_stack_ptr; int i, excnt = 0; - int ilen, proglen = 0; + int ilen, moved, proglen = 0; u8 *ip, *prog = temp; u32 stack_depth; int callee_saved_size; @@ -2700,6 +2661,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * if (rw_image) memcpy(rw_image + proglen, temp, ilen); proglen += ilen; + moved = addrs[0] - proglen; addrs[0] = proglen; prog = temp; @@ -3745,8 +3707,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * default: /* to silence GCC warning */ return -EFAULT; } - jmp_offset = addrs[i + insn->off] - addrs[i]; - if (is_imm8_jmp_offset(jmp_offset)) { + jmp_offset = jmp_distance(addrs, i, insn->off, moved); + if (is_imm8(jmp_offset)) { if (jmp_padding) { /* To keep the jmp_offset valid, the extra bytes are * padded before the jump insn, so we subtract the @@ -3786,23 +3748,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * break; case BPF_JMP | BPF_JA: case BPF_JMP32 | BPF_JA: - if (BPF_CLASS(insn->code) == BPF_JMP) { - if (insn->off == -1) - /* -1 jmp instructions will always jump - * backwards two bytes. Explicitly handling - * this case avoids wasting too many passes - * when there are long sequences of replaced - * dead code. - */ - jmp_offset = -2; - else - jmp_offset = addrs[i + insn->off] - addrs[i]; - } else { - if (insn->imm == -1) - jmp_offset = -2; - else - jmp_offset = addrs[i + insn->imm] - addrs[i]; - } + if (BPF_CLASS(insn->code) == BPF_JMP) + jmp_offset = jmp_distance(addrs, i, insn->off, moved); + else + jmp_offset = jmp_distance(addrs, i, insn->imm, moved); if (!jmp_offset) { /* @@ -3831,7 +3780,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * break; } emit_jmp: - if (is_imm8_jmp_offset(jmp_offset)) { + if (is_imm8(jmp_offset)) { if (jmp_padding) { /* To avoid breaking jmp_offset, the extra bytes * are padded before the actual jmp insn, so @@ -3867,7 +3816,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * emit_unwind_ret_addr(&prog, ip)) return -EINVAL; if (seen_exit) { - jmp_offset = ctx->cleanup_addr - addrs[i]; + jmp_offset = ctx->cleanup_addr - addrs[i] + moved; goto emit_jmp; } seen_exit = true; @@ -3928,6 +3877,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * memcpy(rw_image + proglen, temp, ilen); } proglen += ilen; + moved = addrs[i] - proglen; addrs[i] = proglen; prog = temp; } @@ -4965,20 +4915,23 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr /* * Before first pass, make a rough estimation of addrs[] - * each BPF instruction is translated to less than 64 bytes + * each BPF instruction is translated to at most BPF_MAX_INSN_SIZE + * bytes, so no insn grows in the first pass */ for (proglen = 0, i = 0; i <= prog->len; i++) { - proglen += 64; + proglen += BPF_MAX_INSN_SIZE; addrs[i] = proglen; } ctx.cleanup_addr = proglen; skip_init_addrs: /* - * JITed image shrinks with every pass and the loop iterates - * until the image stops shrinking. Very large BPF programs - * may converge on the last pass. In such case do one more - * pass to emit the final image. + * Jumps are sized by the insns they jump over, which start at + * their maximum size, so no insn grows from one pass to the + * next. The loop iterates until the image stops shrinking, at + * which point no insn has moved. Very large BPF programs may + * converge on the last pass. In such case do one more pass to + * emit the final image. */ for (pass = 0; pass < MAX_PASSES || image; pass++) { if (!padding && pass >= PADDING_PASSES) From a938f51d3522b9ff1fbe1171d9504d584a40eb58 Mon Sep 17 00:00:00 2001 From: Mykyta Yatsenko Date: Thu, 22 Oct 2026 05:04:56 -0700 Subject: [PATCH 08/48] selftests/bpf: Test JIT jumps which change at a repeated image length Cover the case where two JIT passes give the same image length while jumps still change size, which used to fail with -ENOTSUPP on x86-64. Signed-off-by: Mykyta Yatsenko Signed-off-by: Alexei Starovoitov Acked-by: Daniel Borkmann Link: https://patch.msgid.link/20261022-jit_size_fix-v2-2-4bee6921b218@meta.com --- .../selftests/bpf/prog_tests/jit_insns.c | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/jit_insns.c b/tools/testing/selftests/bpf/prog_tests/jit_insns.c index dabefdb8a353..52bffe27199b 100644 --- a/tools/testing/selftests/bpf/prog_tests/jit_insns.c +++ b/tools/testing/selftests/bpf/prog_tests/jit_insns.c @@ -2881,6 +2881,63 @@ static void test_convergence(void) } } +/* + * On x86-64 a backward jump sized from its own previous-pass address grew + * here while others shrank, so two passes gave the same image length with + * jumps still changing. The fillers are 7 and 2 bytes of native code, the + * conditional jumps are never taken. + */ +static void test_same_len_not_converged(void) +{ + struct bpf_insn insns[] = { + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0), + BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1), + BPF_JMP_IMM(BPF_JA, 0, 0, 1), + BPF_JMP_IMM(BPF_JA, 0, 0, 29), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 20), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, -3), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 22), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 2), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 23), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP_IMM(BPF_JA, 0, 0, -22), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 3), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 2), + BPF_JMP_IMM(BPF_JA, 0, 0, -22), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 10), + BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int fd; + + fd = load_prog(insns, ARRAY_SIZE(insns), "same len not converged"); + if (fd < 0) + return; + ASSERT_GT(jited_len(fd), 0, "same len not converged jited"); + prog_run(fd, 1, 2, 11, "same len not converged"); + close(fd); +} + /* * Calls into a subprogram with all five arguments, which clobbers a subset * of r6-r9 and uses the deepest stack slot the budget leaves, directly or @@ -3087,6 +3144,8 @@ void test_jit_insns(void) test_max_jmp(); if (test__start_subtest("convergence")) test_convergence(); + if (test__start_subtest("same_len_not_converged")) + test_same_len_not_converged(); if (test__start_subtest("calls")) test_calls(); if (test__start_subtest("helper_calls")) From b93cf2c8a0552b544e90e25802e4bf989987d29d Mon Sep 17 00:00:00 2001 From: Ayushman Rout Date: Thu, 22 Oct 2026 16:53:11 +0530 Subject: [PATCH 09/48] bpf: Use memmove() to load bytes from an skb and an xdp_buff bpf_dynptr_read() on an skb or xdp dynptr ends in bpf_skb_load_bytes() or bpf_xdp_load_bytes(). The destination can be a slice of the same packet, so it can overlap the source, which memcpy() does not allow. On x86 with FSRM memcpy() copies forward, so a destination above the source gets the first bytes repeated: p = bpf_dynptr_slice_rdwr(&ptr, 8, NULL, 64); bpf_dynptr_read(p, 64, &ptr, 0, 0); leaves bytes 8-71 of the packet as bytes 0-7 repeated, not bytes 0-63. Reproduced with BPF_PROG_TEST_RUN on both an xdp and a tc program. Use memmove(), as the store side does since commit 1db084f42884 ("bpf: Use memmove() to store bytes in an skb") and commit ab08863c58b1 ("bpf: Use memmove() to store bytes in an xdp_buff"). A read that spans xdp frags still goes through bpf_xdp_copy_buf() and is not changed here. bpf_skb_load_bytes_relative() has the same pattern and is reachable from tc and sk_filter programs, so use memmove() there as well. Reproduced with BPF_PROG_TEST_RUN on a tc program: loading into a slice at offset 8 gave wrong bytes before the change and correct bytes after. Fixes: 2012c867c800 ("bpf: verifier: Accept dynptr mem as mem in helpers") Signed-off-by: Ayushman Rout Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261022112312.31563-1-ayushmanrout27@gmail.com --- net/core/filter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/net/core/filter.c b/net/core/filter.c index b7ac145b68b1..09c60a58c82a 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1789,7 +1789,7 @@ BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset, if (unlikely(!ptr)) goto err_clear; if (ptr != to) - memcpy(to, ptr, len); + memmove(to, ptr, len); return 0; err_clear: @@ -1871,7 +1871,7 @@ BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb, ptr = start + offset; if (likely(ptr + len <= end)) { - memcpy(to, ptr, len); + memmove(to, ptr, len); return 0; } @@ -4298,7 +4298,7 @@ BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset, if (!ptr) bpf_xdp_copy_buf(xdp, offset, buf, len, false); else - memcpy(buf, ptr, len); + memmove(buf, ptr, len); return 0; } From 93bd472336ac2424f5f3cf05cbb24e02242343b7 Mon Sep 17 00:00:00 2001 From: Aditya Sharma Date: Thu, 1 Oct 2026 12:54:48 +0530 Subject: [PATCH 10/48] bpf: Remove nmi_uaccess_okay() check in bpf_send_signal_common() nmi_uaccess_okay() takes no task argument and is a statement about current. When commit 6280cf718db0 ("bpf: Implement bpf_send_signal_task() kfunc") made the PF_KTHREAD/PF_EXITING and is_global_init() test the supplied task, nmi_uaccess_okay() was left testing current. As a result bpf_send_signal_task() returns -EPERM whenever the calling context happens to be a kernel thread, regardless of which task the signal is aimed at. The same call with the same target succeeds or fails depending only on what the CPU was running: bpf_send_signal_task() from tp_btf/workqueue_execute_start : -EPERM bpf_send_signal_task() from tp_btf/sys_enter : 0 The check was carried over from bpf_probe_write_user() when bpf_send_signal() was added in commit 8b401f9ed244 ("bpf: implement bpf_send_signal() helper"). There it guards a user memory write, but bpf_send_signal_common() does no user memory access, so the check has nothing to guard here. It is also already a no-op outside x86. Remove it, and drop the part of the comment above it that described the memory access checks, as none remain. Fixes: 6280cf718db0 ("bpf: Implement bpf_send_signal_task() kfunc") Suggested-by: Andrii Nakryiko Signed-off-by: Aditya Sharma Signed-off-by: Alexei Starovoitov Link: https://lore.kernel.org/bpf/20260819124324.43162-1-adi.sharma@zohomail.in/T/#u Link: https://patch.msgid.link/20261001072449.1377267-2-adi.sharma@zohomail.in --- kernel/trace/bpf_trace.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index dfac5e1b8f40..14adfb85fdb2 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -867,15 +867,12 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc siginfo = &info; } - /* Similar to bpf_probe_write_user, task needs to be - * in a sound condition and kernel memory access be - * permitted in order to send signal to the current - * task. + /* + * Similar to bpf_probe_write_user, task needs to be + * in a sound condition. */ if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING))) return -EPERM; - if (unlikely(!nmi_uaccess_okay())) - return -EPERM; /* Task should not be pid=1 to avoid kernel panic. */ if (unlikely(is_global_init(task))) return -EPERM; From 2c79610f72bb880cbb42adacec5a4b1b335040bf Mon Sep 17 00:00:00 2001 From: Aditya Sharma Date: Thu, 1 Oct 2026 12:54:49 +0530 Subject: [PATCH 11/48] selftests/bpf: Test bpf_send_signal_task() from kernel thread context The existing send_signal tests, including the _remote variants that exercise bpf_send_signal_task() with a caller-supplied target, all gate on if ((bpf_get_current_pid_tgid() >> 32) == pid) so the BPF program only ever runs while current is the test process. The caller-context dimension is therefore untested, which is how bpf_send_signal_task() failing from kernel thread context went unnoticed. Add a program attached to tp_btf/workqueue_execute_start, where current is always a kworker. It cannot filter on current, so it is driven by target_pid and gated on a kworker_test flag that only this test sets, and the test asserts the child receives the signal. Setting pid to 0 leaves the pre-existing programs in the skeleton inert. Workqueues run on their own, so nothing has to be triggered from userspace. Without removing the nmi_uaccess_okay() check, this test fails on x86. Signed-off-by: Aditya Sharma Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261001072449.1377267-3-adi.sharma@zohomail.in --- .../selftests/bpf/prog_tests/send_signal.c | 106 ++++++++++++++++++ .../bpf/progs/test_send_signal_kern.c | 29 +++++ 2 files changed, 135 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/send_signal.c b/tools/testing/selftests/bpf/prog_tests/send_signal.c index 7ac4d5a488aa..ecec2ee8b19a 100644 --- a/tools/testing/selftests/bpf/prog_tests/send_signal.c +++ b/tools/testing/selftests/bpf/prog_tests/send_signal.c @@ -261,6 +261,110 @@ static void test_send_signal_nmi(bool signal_thread, bool remote) test_send_signal_common(&attr, signal_thread, remote); } +static void test_send_signal_kworker(void) +{ + struct test_send_signal_kern *skel; + int pipe_c2p[2], pipe_p2c[2]; + struct sigaction sa = {}; + char buf[256]; + int err = -1; + pid_t pid; + + if (!ASSERT_OK(pipe(pipe_c2p), "pipe_c2p")) + return; + + if (!ASSERT_OK(pipe(pipe_p2c), "pipe_p2c")) { + close(pipe_c2p[0]); + close(pipe_c2p[1]); + return; + } + + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) { + close(pipe_c2p[0]); + close(pipe_c2p[1]); + close(pipe_p2c[0]); + close(pipe_p2c[1]); + return; + } + + if (pid == 0) { + /* install signal handler and notify parent */ + sa.sa_sigaction = sigusr1_siginfo_handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + ASSERT_NEQ(sigaction(SIGUSR1, &sa, NULL), -1, "sigaction"); + + close(pipe_c2p[0]); /* close read */ + close(pipe_p2c[1]); /* close write */ + + /* notify parent signal handler is installed */ + ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write"); + + /* make sure parent enabled bpf program to send_signal */ + ASSERT_EQ(read(pipe_p2c[0], buf, 1), 1, "pipe_read"); + + /* + * The signal is sent from workqueue context, so nothing has + * to be triggered from here. + */ + while (!sigusr1_received) + sleep(1); + + buf[0] = sigusr1_received; + + ASSERT_EQ(sigusr1_received, 8, "sigusr1_received"); + ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write"); + + close(pipe_c2p[1]); + close(pipe_p2c[0]); + exit(0); + } + + close(pipe_c2p[1]); /* close write */ + close(pipe_p2c[0]); /* close read */ + + skel = test_send_signal_kern__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + goto skel_open_load_failure; + + /* wait until child signal handler installed */ + ASSERT_EQ(read(pipe_c2p[0], buf, 1), 1, "pipe_read"); + + /* pid == 0 keeps the other programs of this skeleton inactive */ + skel->bss->kworker_test = 1; + skel->bss->pid = 0; + skel->bss->sig = SIGUSR1; + skel->bss->target_pid = pid; + + err = test_send_signal_kern__attach(skel); + if (!ASSERT_OK(err, "skel_attach")) { + err = -1; + goto destroy_skel; + } + + /* notify child that bpf program can send_signal now */ + ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write"); + + /* wait for result, workqueues run on their own */ + err = read_with_timeout(pipe_c2p[0], buf, 1, 10 * 1000 * 1000); + if (!ASSERT_GT(err, 0, "reading pipe")) + goto destroy_skel; + + ASSERT_EQ(buf[0], 8, "incorrect result"); + +destroy_skel: + test_send_signal_kern__destroy(skel); +skel_open_load_failure: + close(pipe_c2p[0]); + close(pipe_p2c[1]); + /* + * Child is either about to exit cleanly or stuck in case of errors. + * Nudge it to exit. + */ + kill(pid, SIGKILL); + wait(NULL); +} + void test_send_signal(void) { if (test__start_subtest("send_signal_tracepoint")) @@ -289,4 +393,6 @@ void test_send_signal(void) test_send_signal_perf(true, true); if (test__start_subtest("send_signal_nmi_thread_remote")) test_send_signal_nmi(true, true); + if (test__start_subtest("send_signal_kworker")) + test_send_signal_kworker(); } diff --git a/tools/testing/selftests/bpf/progs/test_send_signal_kern.c b/tools/testing/selftests/bpf/progs/test_send_signal_kern.c index 176a355e3062..f9c4496d9fe1 100644 --- a/tools/testing/selftests/bpf/progs/test_send_signal_kern.c +++ b/tools/testing/selftests/bpf/progs/test_send_signal_kern.c @@ -9,6 +9,7 @@ void bpf_task_release(struct task_struct *p) __ksym; int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type, u64 value) __ksym; __u32 sig = 0, pid = 0, status = 0, signal_thread = 0, target_pid = 0; +__u32 kworker_test = 0; static __always_inline int bpf_send_signal_test(void *ctx) { @@ -60,6 +61,34 @@ int send_signal_tp_sched(void *ctx) return bpf_send_signal_test(ctx); } +/* + * Send a signal to a task other than current, from a context where current + * is a kernel thread. No filtering on current is possible here, so this is + * driven by target_pid, and gated on kworker_test so that it stays inactive + * in the other tests. + */ +SEC("tp_btf/workqueue_execute_start") +int send_signal_kworker(void *ctx) +{ + struct task_struct *target_task; + int ret; + + if (status != 0 || !kworker_test || target_pid == 0) + return 0; + + target_task = bpf_task_from_pid(target_pid); + if (!target_task) + return 0; + + ret = bpf_send_signal_task(target_task, sig, PIDTYPE_TGID, 8); + bpf_task_release(target_task); + + if (ret == 0) + status = 1; + + return 0; +} + SEC("perf_event") int send_signal_perf(void *ctx) { From d09c2aa3109f5cb1d0be7d382231ab5d58583cad Mon Sep 17 00:00:00 2001 From: Xu Yunxiang Date: Sat, 24 Oct 2026 18:06:46 +0800 Subject: [PATCH 12/48] bpf: Track borrowed iterator results by default An open-coded iterator can return a struct pointer borrowed from its current element. The verifier has not tied such results to iterator state since the task_vma iterator was introduced, so a saved result can still be used after destroy. Commit bee9ef4a40a27 ("bpf: switch task_vma iterator from mmap_lock to per-VMA locks") also made next release the protection of the prior element. A later snapshot conversion made vm_file explicitly borrowed from that element. Associate non-RCU, non-acquired struct results with the iterator reference by default. Invalidate dependents before next forks its NULL and non-NULL paths, while retaining the iterator itself. Let destroy propagate child-reference errors. Add KF_ITER_RET_INDEPENDENT for results that outlive the current element, preserving existing RCU and acquired-reference rules. Trusted pointer field loads also need to preserve the source lifetime. Loading a trusted field does not acquire a reference to the pointed-to object. Set the field pointer's parent_id to the source's reference ID if the source is referenced, or inherit its parent_id if it is borrowed. This is a generic rule, including accesses unrelated to iterators. For example, the following access is now rejected: file = bpf_get_task_exe_file(task); if (!file) return 0; inode = file->f_inode; bpf_put_file(file); ino = inode->i_ino; The inode pointer borrows the file's lifetime. Releasing the acquired file reference removes that lifetime guarantee, so the verifier must invalidate the derived inode pointer as well. This does not imply that bpf_put_file() immediately frees the file or inode. Reading the inode while the file reference is held remains valid. Fields with independent RCU protection retain their existing lifetime rules. For an iterator result, the same propagation keeps trusted fields such as vm_file and f_inode tied to the current element. Propagate the backing object's lifetime through dynptr construction too, so its descendants remain in the invalidation graph. Share the invalidation traversal between next and release, so advancing the iterator preserves its own reference. Fixes: 4ac454682158 ("bpf: Introduce task_vma open-coded iterator kfuncs") Suggested-by: Amery Hung Signed-off-by: Xu Yunxiang Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Link: https://patch.msgid.link/20261024100647.2579244-2-xyx2021@mail.ustc.edu.cn --- Documentation/bpf/bpf_iterators.rst | 10 +++++ include/linux/btf.h | 1 + kernel/bpf/btf.c | 13 +++++- kernel/bpf/verifier.c | 68 ++++++++++++++++++++++++----- 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/Documentation/bpf/bpf_iterators.rst b/Documentation/bpf/bpf_iterators.rst index 189e3ec1c6c8..72435e32ffa1 100644 --- a/Documentation/bpf/bpf_iterators.rst +++ b/Documentation/bpf/bpf_iterators.rst @@ -86,6 +86,16 @@ Constructor/next/destructor implementation contract is as follows: resources and marks stack space used by `struct bpf_iter_` as usable for something else. Destructor is marked with KF_ITER_DESTROY flag. +The verifier treats a non-RCU, non-acquired struct result as borrowed from +the iterator's current element. It invalidates the previous result and its +tracked derived pointers on the next call, including when it returns NULL, +and on destruction of the iterator. Resources borrowing that lifetime must +be released first. A next method whose struct result has an independent +lifetime must use ``KF_ITER_RET_INDEPENDENT``. This flag requires +``KF_ITER_NEXT`` and a struct pointer return type, and cannot be combined +with ``KF_ACQUIRE``. RCU-protected and acquired results retain their own +lifetime rules. + Any open-coded BPF iterator implementation has to implement at least these three methods. It is enforced that for any given type of iterator only applicable constructor/destructor/next are callable. I.e., verifier ensures diff --git a/include/linux/btf.h b/include/linux/btf.h index b66e20a18c54..a18254b7521b 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -82,6 +82,7 @@ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ #define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ #define KF_PERFMON (1 << 18) /* kfunc requires CAP_PERFMON */ +#define KF_ITER_RET_INDEPENDENT (1 << 19) /* next result outlives the current element */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 94d43612d211..7ed1ff71a089 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -9720,6 +9720,10 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, if (!flags || (flags & (flags - 1))) return -EINVAL; + if ((func_flags & KF_ITER_RET_INDEPENDENT) && + (!(flags & KF_ITER_NEXT) || (func_flags & KF_ACQUIRE))) + return -EINVAL; + /* any BPF iter kfunc should have `struct bpf_iter_ *` first arg */ nr_args = btf_type_vlen(func); if (nr_args < 1) @@ -9760,6 +9764,12 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, t = btf_type_skip_modifiers(btf, func->type, NULL); if (!t || !btf_type_is_ptr(t)) return -EINVAL; + /* The opt-out applies only to non-acquired struct results. */ + if (func_flags & KF_ITER_RET_INDEPENDENT) { + t = btf_type_skip_modifiers(btf, t->type, NULL); + if (!t || !__btf_type_is_struct(t)) + return -EINVAL; + } } if (flags & KF_ITER_DESTROY) { @@ -9826,7 +9836,8 @@ static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) if (!func || !btf_type_is_func_proto(func)) return -EINVAL; - if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY)) { + if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY | + KF_ITER_RET_INDEPENDENT)) { err = btf_check_iter_kfuncs(btf, func_name, func, func_flags); if (err) return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5601d2056218..c223bf3bc9cd 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -211,6 +211,9 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par static int __release_reference_nomark(struct bpf_verifier_state *state, int id); static int release_reference_nomark(struct bpf_verifier_env *env, int id); static int release_reference(struct bpf_verifier_env *env, int id); +static int invalidate_reference(struct bpf_verifier_env *env, int id); +static u32 reg_lifetime_id(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); @@ -793,8 +796,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ if (err) return err; - /* Track parent's id if the parent is a referenced object */ - parent_id = ref_obj->id; + /* All non-clone constructors take their backing object in R1. */ + parent_id = reg_lifetime_id(env, &cur_regs(env)[BPF_REG_1]); if (dynptr_type_referenced(type)) { int id; @@ -1074,7 +1077,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) { struct bpf_func_state *state = bpf_func(env, reg); - int spi, i, j; + int spi, i, j, err; spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) @@ -1084,8 +1087,11 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; - if (i == 0) - WARN_ON_ONCE(release_reference(env, st->id)); + if (i == 0) { + err = release_reference(env, st->id); + if (err) + return err; + } bpf_mark_reg_not_init(env, st); @@ -1627,6 +1633,12 @@ static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg return find_reference_state(env->cur_state, reg->id); } +static u32 reg_lifetime_id(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + return reg_is_referenced(env, reg) ? reg->id : reg->parent_id; +} + static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) { struct bpf_verifier_state *state = env->cur_state; @@ -6848,9 +6860,14 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, flag &= ~PTR_MAYBE_NULL; if (atype == BPF_READ && value_regno >= 0) { + u32 parent_id = reg_lifetime_id(env, reg); + ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) return ret; + if ((regs[value_regno].type & PTR_TRUSTED) && + !(regs[value_regno].type & (MEM_RCU | MEM_PERCPU | MEM_USER))) + regs[value_regno].parent_id = parent_id; } return 0; @@ -8636,6 +8653,21 @@ static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_ITER_DESTROY; } +static bool iter_next_borrows_struct(struct bpf_call_arg_meta *meta) +{ + const struct btf_type *t; + + if (!(meta->kfunc_flags & KF_ITER_NEXT) || + (meta->kfunc_flags & (KF_ITER_RET_INDEPENDENT | KF_ACQUIRE))) + return false; + + t = btf_type_skip_modifiers(meta->btf, meta->func_proto->type, NULL); + if (!t || !btf_type_is_ptr(t)) + return false; + t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + return t && __btf_type_is_struct(t); +} + static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, const struct btf_param *arg) { @@ -8746,6 +8778,13 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * meta->iter.frameno = reg->frameno; update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); + /* The iterator stays. What was borrowed from the previous element goes. */ + if (iter_next_borrows_struct(meta)) { + err = invalidate_reference(env, meta->ref_obj.id); + if (err) + return err; + } + if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); if (err) @@ -11073,8 +11112,8 @@ static int idstack_pop(struct bpf_idmap *idmap) return idmap->map[--idmap->cnt].old; } -/* Release id and objects derived from it iteratively in a DFS manner */ -static int release_reference(struct bpf_verifier_env *env, int id) +/* Invalidate id and objects derived from it iteratively in a DFS manner */ +static int invalidate_reference(struct bpf_verifier_env *env, int id) { u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR); struct bpf_verifier_state *vstate = env->cur_state; @@ -11089,11 +11128,6 @@ static int release_reference(struct bpf_verifier_env *env, int id) if (err) return err; - if (find_reference_state(vstate, id)) { - err = release_reference_nomark(env, id); - WARN_ON_ONCE(err); - } - while ((id = idstack_pop(idstack))) { /* * Child references are inaccessible after parent is released, @@ -11145,6 +11179,14 @@ static int release_reference(struct bpf_verifier_env *env, int id) return 0; } +static int release_reference(struct bpf_verifier_env *env, int id) +{ + if (find_reference_state(env->cur_state, id)) + WARN_ON_ONCE(release_reference_nomark(env, id)); + + return invalidate_reference(env, id); +} + static void invalidate_non_owning_refs(struct bpf_verifier_env *env) { struct bpf_func_state *unused; @@ -15822,6 +15864,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].type = type; regs[BPF_REG_0].btf_id = ptr_type_id; + if (iter_next_borrows_struct(&meta) && !(type & MEM_RCU)) + regs[BPF_REG_0].parent_id = meta.ref_obj.id; } if (is_kfunc_ret_null(&meta)) { From 2d7eaeb05ae265bd64270376e6a2c0d80244771f Mon Sep 17 00:00:00 2001 From: Xu Yunxiang Date: Sat, 24 Oct 2026 18:06:47 +0800 Subject: [PATCH 13/48] selftests/bpf: Test borrowed iterator result lifetimes Exercise borrowed VMA results before and after iterator advancement and destruction, including NULL next, trusted fields, and a FILE dynptr that must be discarded before advancing. Keep positive controls for using the current element and discarding the dynptr first. Use a test-module iterator returning current to verify the explicit independent-lifetime exception. Retain controls for RCU-protected results. Also check the generic trusted-field rule outside iterators: reading file->f_inode while holding an acquired file reference succeeds, but using the saved inode pointer after bpf_put_file() is rejected. The annotated programs check verifier loading and diagnostics; they do not request runtime execution of the added programs. Signed-off-by: Xu Yunxiang Signed-off-by: Alexei Starovoitov Reviewed-by: Amery Hung Link: https://patch.msgid.link/20261024100647.2579244-3-xyx2021@mail.ustc.edu.cn --- .../selftests/bpf/progs/iters_testmod.c | 370 ++++++++++++++++++ .../selftests/bpf/progs/verifier_vfs_accept.c | 17 + .../selftests/bpf/progs/verifier_vfs_reject.c | 19 + .../selftests/bpf/test_kmods/bpf_testmod.c | 25 ++ .../selftests/bpf/test_kmods/bpf_testmod.h | 4 + .../bpf/test_kmods/bpf_testmod_kfunc.h | 5 + 6 files changed, 440 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index 145750bb64ef..8896335668b4 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c @@ -8,6 +8,44 @@ char _license[] SEC("license") = "GPL"; +struct bpf_iter_testmod_current { + u64 __opaque; +}; + +SEC("raw_tp/sys_enter") +__success +int iter_independent_lifetime_after_next(const void *ctx) +{ + struct bpf_iter_testmod_current it; + struct task_struct *task; + + bpf_iter_testmod_current_new(&it); + task = bpf_iter_testmod_current_next(&it); + if (!task) + goto out; + + bpf_iter_testmod_current_next(&it); + bpf_kfunc_trusted_task_test(task); +out: + bpf_iter_testmod_current_destroy(&it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_independent_lifetime_after_destroy(const void *ctx) +{ + struct bpf_iter_testmod_current it; + struct task_struct *task; + + bpf_iter_testmod_current_new(&it); + task = bpf_iter_testmod_current_next(&it); + bpf_iter_testmod_current_destroy(&it); + if (task) + bpf_kfunc_trusted_task_test(task); + return 0; +} + SEC("raw_tp/sys_enter") __success int iter_next_trusted(const void *ctx) @@ -28,6 +66,298 @@ int iter_next_trusted(const void *ctx) return 0; } +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + bpf_iter_task_vma_destroy(&vma_it); + return vma_ptr->vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + u64 vm_start; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!next_vma_ptr) + goto out; + + vm_start = vma_ptr->vm_start; + bpf_iter_task_vma_destroy(&vma_it); + return vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_next_null(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + u64 vm_start; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (next_vma_ptr) + goto out; + + vm_start = vma_ptr->vm_start; + bpf_iter_task_vma_destroy(&vma_it); + return vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_field_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + bpf_iter_task_vma_destroy(&vma_it); + if (file_ptr) + return file_ptr->f_mode; + return 0; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_field_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + struct file *file_ptr; + u32 mode; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!next_vma_ptr) + goto out; + + mode = file_ptr->f_mode; + bpf_iter_task_vma_destroy(&vma_it); + return mode; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("Leaking reference id=") +int iter_next_file_dynptr_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_iter_task_vma_next(&vma_it); + bpf_dynptr_file_discard(&dynptr); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("Leaking reference id=") +int iter_next_file_dynptr_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_iter_task_vma_destroy(&vma_it); + bpf_dynptr_file_discard(&dynptr); + return 0; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_file_dynptr_discard_before_advance(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_dynptr_file_discard(&dynptr); + bpf_iter_task_vma_next(&vma_it); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_current_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) + bpf_kfunc_trusted_vma_test(vma_ptr); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_rcu_field_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct mm_struct *mm_ptr; + struct file *file_ptr = NULL; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) { + mm_ptr = vma_ptr->vm_mm; + /* exe_file has RCU protection independent of the iterator. */ + if (mm_ptr) + file_ptr = mm_ptr->exe_file; + } + + bpf_iter_task_vma_destroy(&vma_it); + if (file_ptr) + return file_ptr->f_mode; + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_rcu_field_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct mm_struct *mm_ptr; + struct file *file_ptr = NULL; + u32 mode = 0; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) { + mm_ptr = vma_ptr->vm_mm; + if (mm_ptr) + file_ptr = mm_ptr->exe_file; + } + + bpf_iter_task_vma_next(&vma_it); + if (file_ptr) + mode = file_ptr->f_mode; + bpf_iter_task_vma_destroy(&vma_it); + return mode; +} + SEC("raw_tp/sys_enter") __failure __msg("Possibly NULL pointer passed to trusted R1") int iter_next_trusted_or_null(const void *ctx) @@ -77,6 +407,46 @@ int iter_next_rcu(const void *ctx) return 0; } +SEC("raw_tp/sys_enter") +__success +int iter_next_rcu_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task task_it; + struct task_struct *task_ptr; + + bpf_iter_task_new(&task_it, cur_task, 0); + + task_ptr = bpf_iter_task_next(&task_it); + bpf_iter_task_destroy(&task_it); + if (!task_ptr) + return 0; + + bpf_kfunc_rcu_task_test(task_ptr); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_rcu_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task task_it; + struct task_struct *task_ptr; + + bpf_iter_task_new(&task_it, cur_task, 0); + + task_ptr = bpf_iter_task_next(&task_it); + if (!task_ptr) + goto out; + + bpf_iter_task_next(&task_it); + bpf_kfunc_rcu_task_test(task_ptr); +out: + bpf_iter_task_destroy(&task_it); + return 0; +} + SEC("raw_tp/sys_enter") __failure __msg("Possibly NULL pointer passed to trusted R1") int iter_next_rcu_or_null(const void *ctx) diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c b/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c index 55398c04290a..25f37a3dd7cf 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c @@ -11,6 +11,23 @@ static char buf[64]; +SEC("lsm.s/file_open") +__success +int BPF_PROG(inode_field_before_file_put) +{ + struct file *file; + ino_t ino; + + file = bpf_get_task_exe_file(bpf_get_current_task_btf()); + if (!file) + return 0; + + ino = file->f_inode->i_ino; + bpf_put_file(file); + __sink(ino); + return 0; +} + SEC("lsm.s/file_open") __success int BPF_PROG(get_task_exe_file_and_put_kfunc_from_current_sleepable) diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index b5f456d57669..ed2bd2faebf3 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -12,6 +12,25 @@ static char buf[PATH_MAX]; +SEC("lsm.s/file_open") +__failure __msg("invalid mem access 'scalar'") +int BPF_PROG(inode_field_after_file_put) +{ + struct inode *inode; + struct file *file; + ino_t ino; + + file = bpf_get_task_exe_file(bpf_get_current_task_btf()); + if (!file) + return 0; + + inode = file->f_inode; + bpf_put_file(file); + ino = inode->i_ino; + __sink(ino); + return 0; +} + SEC("lsm.s/file_open") __failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(get_task_exe_file_kfunc_null) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 6380f52f4dd2..76fbe64755ef 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -281,6 +281,27 @@ __bpf_kfunc void bpf_iter_testmod_seq_destroy(struct bpf_iter_testmod_seq *it) it->cnt = 0; } +__bpf_kfunc int bpf_iter_testmod_current_new(struct bpf_iter_testmod_current *it) +{ + it->remaining = 2; + return 0; +} + +__bpf_kfunc struct task_struct * +bpf_iter_testmod_current_next(struct bpf_iter_testmod_current *it) +{ + if (!it->remaining) + return NULL; + it->remaining--; + /* current remains alive independently of the iterator. */ + return current; +} + +__bpf_kfunc void bpf_iter_testmod_current_destroy(struct bpf_iter_testmod_current *it) +{ + it->remaining = 0; +} + __bpf_kfunc void bpf_kfunc_common_test(void) { } @@ -1016,6 +1037,10 @@ BTF_ID_FLAGS(func, bpf_iter_testmod_seq_new, KF_ITER_NEW) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_value) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_new, KF_ITER_NEW) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_next, + KF_ITER_NEXT | KF_RET_NULL | KF_ITER_RET_INDEPENDENT) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_kfunc_common_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_arg_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_test) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 4ba9def74404..aef780261ef4 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -31,6 +31,10 @@ struct bpf_iter_testmod_seq { int cnt; }; +struct bpf_iter_testmod_current { + u64 remaining; +}; + struct bpf_testmod_ops { int (*test_1)(void); void (*test_2)(int a, int b); diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 3b810f1e438f..e9902e651d72 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -33,6 +33,11 @@ struct prog_test_ref_kfunc { #endif struct bpf_iter_testmod_seq; +struct bpf_iter_testmod_current; + +int bpf_iter_testmod_current_new(struct bpf_iter_testmod_current *it) __ksym; +struct task_struct *bpf_iter_testmod_current_next(struct bpf_iter_testmod_current *it) __ksym; +void bpf_iter_testmod_current_destroy(struct bpf_iter_testmod_current *it) __ksym; struct prog_test_pass1 { int x0; From 90aedf8854c05713c02b381c41c59081b5cedbcc Mon Sep 17 00:00:00 2001 From: Weiming Shi Date: Sun, 25 Oct 2026 18:52:49 -0700 Subject: [PATCH 14/48] bpf: Validate stack arg reads against caller state check_stack_arg_read() bounds a stack argument index with the callee's BTF-derived argument count, then uses the index in the caller's stack_arg_regs[] array. The array's actual logical length is recorded in caller->out_stack_arg_cnt, so the bound and indexed object can diverge. A crafted BPF_PROG_LOAD can use an argument mismatch to mark a static subprogram's BTF information unreliable. Later calls return from btf_check_subprog_call() before check_outgoing_stack_args(), while check_func_call() continues the static call on -EINVAL. The callee can then read a slot that the caller never allocated or initialized. The earlier commit 4286f5deee14 ("bpf: Validate outgoing stack args when btf_prepare_func_args fails") closed this hole for the first call, but not for later calls after the subprogram has been marked unreliable. Callbacks have a related problem. The verifier can make incoming stack arguments readable from outgoing slots at the helper call site. However, helpers and kfuncs invoke callbacks through an ABI that passes at most five arguments, so the runtime callback receives no stack arguments. Async callback verification can also reach check_stack_arg_read() without a caller frame. On x86-64, KASAN reports the direct-call out-of-bounds copy as: BUG: KASAN: slab-out-of-bounds in check_load_mem+0xb51/0xc60 Read of size 80 at addr ffff8881009af550 by task poc/131 Call Trace: check_stack_arg_read at kernel/bpf/verifier.c:4234 (inlined by) check_load_mem at kernel/bpf/verifier.c:6645 do_check_common at kernel/bpf/verifier.c:19568 bpf_check at kernel/bpf/verifier.c:21318 bpf_prog_load at kernel/bpf/syscall.c:3134 Reject callback subprograms with incoming stack arguments before their state is pushed. For direct calls, retrieve the slot with bpf_get_spilled_stack_arg(), which validates the index against the caller's out_stack_arg_cnt and rejects uninitialized slots before copying the register state. Fixes: 0f6bd5e7a804 ("bpf: Support stack arguments for bpf functions") Reported-by: Signed-off-by: Weiming Shi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026015250.1674676-1-bestswngs@gmail.com --- kernel/bpf/verifier.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c223bf3bc9cd..a0a5f075d201 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4666,7 +4666,12 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st } caller = vstate->frame[vstate->curframe - 1]; - arg = &caller->stack_arg_regs[spi]; + arg = bpf_get_spilled_stack_arg(spi, caller); + if (!arg) { + verbose(env, "invalid read from uninitialized stack arg off %d\n", off); + return -EACCES; + } + cur = vstate->frame[vstate->curframe]; bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE); cur->regs[dst_regno] = *arg; @@ -11451,6 +11456,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; + if (bpf_in_stack_arg_cnt(&env->subprog_info[subprog])) { + verbose(env, "callback subprog cannot have stack args\n"); + return -EINVAL; + } /* set_callee_state is used for direct subprog calls, but we are * interested in validating only BPF helpers that can call subprogs as From 195a3519ff27bff5575e05b5086e38ee92715776 Mon Sep 17 00:00:00 2001 From: Weiming Shi Date: Sun, 25 Oct 2026 18:52:50 -0700 Subject: [PATCH 15/48] selftests/bpf: Test missing and callback stack arguments Exercise the path where a static seven-argument subprogram has unreliable BTF and is called through a bridge with only one initialized outgoing stack argument. The callee expects two stack arguments, but the unreliable-BTF path skips check_outgoing_stack_args(). Expect the verifier to reject the second r11 load at offset 16. Also exercise synchronous and asynchronous callbacks whose BTF prototypes declare a sixth argument. Initialize the outgoing slot at each helper call site so the programs reach the callback-specific rejection. Keep a valid six-argument direct call in each program so bpf_fixup_call_args() records the stack-argument requirement and cannot become an unrelated rejection fallback. The fix also rejects the existing uninitialized-stack-argument test before the later unreadable-register check. Update that test's expected diagnostic accordingly. Signed-off-by: Weiming Shi Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026015250.1674676-2-bestswngs@gmail.com --- .../bpf/progs/btf__verifier_stack_arg_order.c | 10 ++ .../bpf/progs/verifier_stack_arg_order.c | 121 +++++++++++++++++- 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c index a980a3a399d3..f4fb8f81afa7 100644 --- a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c @@ -26,6 +26,11 @@ void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) { } +int callback_6args(long a, long b, long c, long d, long e, long f) +{ + return a + b + c + d + e + f; +} + #else int subprog_bad_order_6args(void) @@ -47,4 +52,9 @@ void subprog_bad_ptr_7args(void) { } +int callback_6args(void) +{ + return 0; +} + #endif diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index ab1955852233..5bfcd300c7a0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -5,6 +5,17 @@ #include #include "bpf_misc.h" +struct timer_value { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct timer_value); +} timer_map SEC(".maps"); + #if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) @@ -130,6 +141,25 @@ static void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, in ); } +__noinline __used __naked +static int callback_6args(long a, long b, long c, long d, long e, long f) +{ + asm volatile ("r0 = *(u64 *)(r11 + 8);" + "exit;" + ::: __clobber_all + ); +} + +__noinline __used __naked +static void subprog_call_bad_ptr_7args(void) +{ + asm volatile ("*(u64 *)(r11 - 8) = 0;" + "call subprog_bad_ptr_7args;" + "exit;" + ::: __clobber_all + ); +} + SEC("tc") __description("stack_arg: read without caller write") __failure @@ -152,7 +182,7 @@ __naked void stack_arg_read_without_write_1(void) SEC("tc") __description("stack_arg: read with not-initialized caller write") __failure -__msg("R0 !read_ok") +__msg("invalid read from uninitialized stack arg off 8") __btf_func_path("btf__verifier_stack_arg_order.bpf.o") __naked void stack_arg_read_without_write_2(void) { @@ -171,6 +201,95 @@ __naked void stack_arg_read_without_write_2(void) ); } +SEC("tc") +__description("stack_arg: unreliable callee with too few stack args") +__failure +__msg("invalid read from uninitialized stack arg off 16") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_read_with_short_bridge_write(void) +{ + asm volatile ("r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "*(u64 *)(r11 - 8) = 0;" + "*(u64 *)(r11 - 16) = 0;" + "call subprog_bad_ptr_7args;" + "call subprog_call_bad_ptr_7args;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: synchronous callback with stack args") +__failure +__msg("callback subprog cannot have stack args") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_sync_callback(void) +{ + asm volatile ("r6 = 0;" + "*(u64 *)(r11 - 8) = r6;" + "r1 = 1;" + "r2 = %[callback_6args];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call callback_6args;" + "r0 = 0;" + "exit;" + : + : __imm_ptr(callback_6args), + __imm(bpf_loop) + : __clobber_common, "r6" + ); +} + +SEC("tc") +__description("stack_arg: asynchronous callback with stack args") +__failure +__msg("callback subprog cannot have stack args") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_async_callback(void) +{ + asm volatile ("r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r1 = %[timer_map] ll;" + "r2 = r10;" + "r2 += -4;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l0_%=;" + "r6 = r0;" + "r7 = 0;" + "*(u64 *)(r11 - 8) = r7;" + "r1 = r6;" + "r2 = %[callback_6args];" + "call %[bpf_timer_set_callback];" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call callback_6args;" + "l0_%=: r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(callback_6args), + __imm(bpf_map_lookup_elem), + __imm(bpf_timer_set_callback) + : __clobber_common, "r6", "r7" + ); +} + #else SEC("socket") From c463b378eae7fa4d8013d3ba7ca224f3a5098ec4 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:21 -0700 Subject: [PATCH 16/48] bpf: Reject pointer operands for 32-bit recurrences/latches Avoid inferring loop bounds for the following program shapes: 32-bit latch (A) | 32-bit recurrence (B) -------------------------+-------------------------- r1 = ... pointer ... | r1 = ... pointer ... r2 = ... pointer ... | r2 = ... pointer ... l: r1 += 8 | l: w1 += 8 ... | ... if w1 < w2 goto l | if r1 < r2 goto l (A) Scalar bounds for pointers represent only offsets from their common origin values. The 32-bit operations also depend on the runtime base address: the addresses can wrap at 32 bits without the tracked offsets wrapping. Thus, offsets alone do not determine the iteration count. (B) 32-bit operations on pointers either prohibited (wX += 1) or convert them to scalars (wX = wX). No need to deal with this at SCEV level. Fixes: 32e962577d2d ("bpf: Support 32-bit linear scalar evolution") Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-1-f373aec9825f@gmail.com --- kernel/bpf/scev.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index fea77772ca96..f9e2ca6a5a12 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2024,7 +2024,8 @@ static void log_incompatible_scev_values(struct bpf_verifier_env *env, const cha } static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_state *st, - const struct scev_value *a, const struct scev_value *b) + const struct scev_value *a, const struct scev_value *b, + const struct linear_latch *latch) { const struct bpf_reg_state *ra, *rb; @@ -2035,7 +2036,7 @@ static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_ } ra = scev_reg_for_read(env, st, a->ptr_reg); rb = scev_reg_for_read(env, st, b->ptr_reg); - if (ra && rb && bpf_same_memory_origin(ra, rb)) + if (ra && rb && bpf_same_memory_origin(ra, rb) && !latch->zext32 && !latch->jmp32) return true; incompatible: @@ -2267,7 +2268,7 @@ static bool compute_max_iters(struct bpf_verifier_env *env, } return false; } - if (!scev_value_compatible(env, st, &initial_val, &bound_val)) + if (!scev_value_compatible(env, st, &initial_val, &bound_val, latch)) return false; initial = initial_val.value; @@ -2700,6 +2701,9 @@ int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_sta reg = scev_reg_for_linear(cur_func, r); /* If SCEV for r is (linear ) */ if (reg && is_simple_linear(scev, r_scev, &coefs) && is_widenable_reg_type(reg)) { + /* 32-bit ALU operations on pointers convert them to scalars */ + if (coefs.zext32 && reg->type != SCALAR_VALUE) + goto cant_widen; /* Spills at varying offsets lose precision */ if (test_bit(r, loop->store_base_regs)) { spill_base = true; From 0c33d612bdd50aaeef263410739060827377d9c0 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:22 -0700 Subject: [PATCH 17/48] bpf: Do not intersect with current register range on clamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scev.c:bpf_clamp_scev_regs() is called each time verifier returns to a loop header via a backedge. It's purpose is to re-establish computed variable ranges for registers with linear SCEVs. It can be defined as: - For each register R with linear SCEV compute: - loop_range(R) := entry_range(R) * iterations_count - range(R) := range(R) ∩ loop_range(R) Such formulation is necessary to avoid ranges over-approximation in examples like below: r1 = 0; SCEVs: r1 = (linear r1 1) r3 = 0; r3 = (linear r3 1) loop: r3 += 1; r1 += 1; if r1 < 5 goto loop; r0 = 0; exit; - At loop entry r3 is widened to be in a range r3 ∈ {0,1,2,3,4}. - After increment it becomes r3 ∈ {1,2,3,4,5}. - On a second iteration the bpf_clamp_scev_regs() forces r3 to become r3 ∈ {1,2,3,4}. W/o clamping logic r3 range wouldn't shrink preventing loop from converging in states.c:states_equal(). However, intersection logic might lead to assigning an empty range to a register, as in example below: r1 = 0; verification r3 = 0; path loop: r3 += 1; | <---------. if r3 != 5 goto +0; | r3 ∈ {5} | r1 += 1; | | if r1 != 5 goto loop; '-----------' r0 = 0; exit; - At header r3 ∈ {0,1,2,3,4}. - While exploring the branch `if r3 != 5 goto ...` with r3 ∈ {5}. - On a second iteration: r3 := {0,1,2,3,4} ∩ {5} = ∅ Which stops verification with the following error: verifier bug: scev clamp produced empty range for r3 Technically this means that the current verification path is impossible at runtime and should not be traced further. This commit mitigates the above by removing the intersection logic, the function is now defined as: - For each register R with linear SCEV compute: - range(R) := entry_range(R) * iterations_count This is still sound, provided that SCEV expressions loop execution counts are correct. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-2-f373aec9825f@gmail.com --- kernel/bpf/scev.c | 6 ------ tools/testing/selftests/bpf/progs/verifier_scev.c | 2 +- tools/testing/selftests/bpf/progs/verifier_stack_ptr.c | 7 ++----- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index f9e2ca6a5a12..d67e793d7a18 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2896,12 +2896,6 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur continue; linear_bounds(entry_reg, max_header_count, &coefs, &bounds); - bounds.r64 = cnum64_intersect(reg->r64, bounds.r64); - bounds.r32 = cnum32_intersect(reg->r32, bounds.r32); - if (cnum64_is_empty(bounds.r64)) { - verifier_bug(env, "scev clamp produced empty range for r%d", r); - return -EFAULT; - } if (log->level & BPF_LOG_LEVEL2) { bpf_log(log, "loop header at %d, clamping ", insn_idx); log_reg(env, r); diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 661ce9ca8f93..14594ba5dbfd 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -1592,7 +1592,7 @@ __msg_next(" r0=(+ r0 -1) / (linear (+ r0 -1) -1)") __msg("loop header at 1, header_count is 3 ") __msg("loop header at 1, widening r0 to 1..3 step 1") __msg("1: R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,var_off=(0x0; 0x3)) loop_stack=1") -__msg("loop header at 1, clamping r0 to 1..2 (s32 1..2) step 1") +__msg("loop header at 1, clamping r0 to 1..3 step 1") __msg("processed 5 insns") __naked void post_cond_jne_neg_step(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index 7fa7bc207850..dc176f407c36 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -601,11 +601,8 @@ __flag(BPF_F_TEST_STATE_FREQ) */ __msg("loop header at 3, widening r7 to -72..-16 step 8") __msg("R7=fp(smin=smin32=-64,smax=smax32=-8") -/* - * back-edge state is clamped to the remaining iterations and pruned - * at the header, because [-64, -16] is within the widened [-72, -16] - */ -__msg("loop header at 3, clamping r7 to -64..-16 (s32 -64..-8) step 8") +/* back-edge state is clamped to the widened range and pruned at the header */ +__msg("loop header at 3, clamping r7 to -72..-16 step 8") __msg("from 5 to 3: safe") __msg("processed 9 insns") __naked void stack_ptr_subrange_in_loop(void) From 5dd32c8c108226912054e293ee306c98c283c512 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:23 -0700 Subject: [PATCH 18/48] bpf: Avoid false infinite loop reports after changes in SCEV clamping Clamping now reproduces the widened range at every back edge, so induction variable ranges no longer vary between iterations. This can produce identical checkpoint states in terminating programs, for example: int main(...) { for (int i = 0; i < 10; i++) { foo(...); for (int j = 0; j < 3; j++) ...; } } void foo(...) { ...; } Handle two cases in states.c:bpf_is_state_visited(): - For checkpoints inside foo(), look for active loops in upper frames as well. Otherwise current_loop() returns NULL and an identical callee state can trigger an infinite loop report. - For checkpoints inside the inner loop, skip the infinite loop check when the current state is inside a terminating loop in any frame. That loop bounds how many times its body is re-executed, so an identical state on a later iteration does not imply nontermination. This covers the inner loop above, re-entered with a fresh entry state on each outer iteration, and inner loops SCEV cannot analyze. The nested_terminating_prune test still rejects such an outer loop, but now reaches the jump complexity limit instead of reporting an infinite loop. Update its expected message accordingly. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-3-f373aec9825f@gmail.com --- kernel/bpf/states.c | 36 ++++++++++++++----- .../selftests/bpf/progs/verifier_scev.c | 2 +- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index d7df6632a5c4..a6ae86f973e3 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1358,13 +1358,33 @@ static bool need_loop_checkpoint(struct bpf_verifier_env *env, int insn_idx) return top->loop_id == insn_idx && top->terminates; } +/* Searches for an innermost loop entry starting from an innermost frame. */ static struct loop_stack_entry *current_loop(struct bpf_verifier_state *st) { - struct bpf_func_state *frame = st->frame[st->curframe]; + struct bpf_func_state *frame; + int fr; - if (frame->loop_stack_cnt == 0) - return NULL; - return &frame->loop_stack[frame->loop_stack_cnt - 1]; + for (fr = st->curframe; fr >= 0; fr--) { + frame = st->frame[fr]; + if (frame->loop_stack_cnt) + return &frame->loop_stack[frame->loop_stack_cnt - 1]; + } + return NULL; +} + +/* True if any of the loops call chain is currently in terminates. */ +static bool in_a_terminating_loop(struct bpf_verifier_state *st) +{ + struct bpf_func_state *frame; + int l, fr; + + for (fr = st->curframe; fr >= 0; fr--) { + frame = st->frame[fr]; + for (l = frame->loop_stack_cnt - 1; l >= 0; l--) + if (frame->loop_stack[l].terminates) + return true; + } + return false; } int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) @@ -1528,10 +1548,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ old_loop = current_loop(&sl->state); cur_loop = current_loop(cur); - if (old_loop && cur_loop && - old_loop->terminates && cur_loop->terminates && - old_loop->entry_state == cur_loop->entry_state) { - if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + if (old_loop && cur_loop && in_a_terminating_loop(cur)) { + if (old_loop->terminates && cur_loop->terminates && + old_loop->entry_state == cur_loop->entry_state && + states_equal(env, &sl->state, cur, RANGE_WITHIN)) { loop = true; goto hit; } diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 14594ba5dbfd..9074b2f043d4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3351,7 +3351,7 @@ SEC("xdp") __failure __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) -__msg("infinite loop detected at insn 6") +__msg("The sequence of 8193 jumps is too complex.") __naked void nested_terminating_prune(void) { asm volatile (" \ From 86d654957d40111ec158aae38d7f7b0bc119533c Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:24 -0700 Subject: [PATCH 19/48] bpf: Check the iteration count before the predicted exit value The comment before `total_change` computation: /* * compute_backedge_taken_count() guarantees that 'total_change' * does not overflow in 64-bit domain. */ total_change = initial + step * backedge_taken_count; Is not true before the check: if (backedge_taken_count >= U32_MAX - 1) return false; Move the check before `total_change` computation in order to reconcile the code with comments. Old position is not a bug the check was performed either way. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-4-f373aec9825f@gmail.com --- kernel/bpf/scev.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index d67e793d7a18..e3c2220f89c9 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2307,11 +2307,15 @@ static bool compute_max_iters(struct bpf_verifier_env *env, if (!compute_backedge_taken_count(op, initial, step, bound, &backedge_taken_count)) return false; + /* + * max_header_count is u32 and U32_MAX means "infinite", + * check to avoid overflow below. + */ + if (backedge_taken_count >= U32_MAX - 1) + return false; + if (latch->zext32 || latch->jmp32) { - /* - * compute_backedge_taken_count() guarantees that 'total_change' - * does not overflow in 64-bit domain. - */ + /* total_change is guaranteed not to overflow at this point */ total_change = initial + step * backedge_taken_count; if (latch->jmp32 && is_signed_op(op)) { if ((s64)total_change < S32_MIN || (s64)total_change > S32_MAX) @@ -2322,13 +2326,6 @@ static bool compute_max_iters(struct bpf_verifier_env *env, } } - /* - * max_header_count is u32 and U32_MAX means "infinite", - * check to avoid overflow below. - */ - if (backedge_taken_count >= U32_MAX - 1) - return false; - /* The header executes once on entry and once for each taken backedge, hence +1. */ iters->max_header_count = backedge_taken_count + 1; /* From abc9a76a888f813b698f5e2a0166ace6a0d976c6 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:25 -0700 Subject: [PATCH 20/48] bpf: (zext32 (zext32 X)) -> (zext32 X) SCEV simplification rule The compiler can preserve an induction variable across a call by copying it to a callee-saved register. For example: w1 = 0; loop: w7 = w1; call foo; w1 = w7; w1 += 2; if w7 < 10 goto loop; When building expressions for the loop body, each 32-bit move adds a zero extension: w7 = w1; r7 = (zext32 r1) w1 = w7; r1 = (zext32 (zext32 r1)) w1 += 2; r1 = (zext32 (+ (zext32 (zext32 r1)) 2)) The existing ALU32 simplification rules allow recognizing the header SCEV for r1 as: (zext32 (linear r1 2)) Substituting this SCEV into r7's expression at the latch produces: (zext32 (zext32 (linear r1 2))) Add the simplification rule: (zext32 (zext32 X)) -> (zext32 X) This allows recognizing the saved induction variable in r7 as a 32-bit linear SCEV and computing the loop iteration bound. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-5-f373aec9825f@gmail.com --- kernel/bpf/scev.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index e3c2220f89c9..12b147faa32a 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -1502,6 +1502,10 @@ static int simplify(struct scev *scev, u32 id, void *priv) return t; } + /* (zext32 (zext32 X)) -> (zext32 X) */ + if (is_zext32(scev, id, &a) && is_zext32(scev, a, &b)) + return a; + return id; } From 899cbaae6e4c6f0a1dee01426b53e71c58d396c4 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:26 -0700 Subject: [PATCH 21/48] bpf: Print 32-bit SCEV bounds as 32-bit numbers - Print large 32-bit bounds without sign extension. - Omit 32-bit bounds when they coincide with 64-bit bounds. Before: loop header at ..., widening r0 to 0x80000000..0xffffffff (s32 0xffffffff80000000..-1) step 1 loop header at ..., widening r0 to 0..3 (s32 0..3) step 1 After: loop header at ..., widening r0 to 0x80000000..0xffffffff (s32 0x80000000..-1) step 1 loop header at ..., widening r0 to 0..3 step 1 Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-6-f373aec9825f@gmail.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/log.c | 8 ++++++++ kernel/bpf/scev.c | 8 +++++--- tools/testing/selftests/bpf/progs/verifier_scev.c | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5926f13ff43c..e51984cb3035 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1280,6 +1280,7 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...); void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num); +void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num); #define verifier_bug_if(cond, env, fmt, args...) \ ({ \ diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 5432c0fc442b..fa29d9b994e4 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -535,6 +535,14 @@ void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num) verbose(env, "%#llx", num); } +void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num) +{ + if (is_snum_decimal(num)) + verbose(env, "%d", num); + else + verbose(env, "%#x", (u32)num); +} + int tnum_strn(char *str, size_t size, struct tnum a) { /* print as a constant, if tnum is fully known */ diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 12b147faa32a..04d4962448e2 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2456,11 +2456,13 @@ static void log_bounds(struct bpf_verifier_env *env, const struct bounds *bounds bpf_verbose_snum(env, cnum64_smin(bounds->r64)); bpf_log(log, ".."); bpf_verbose_snum(env, cnum64_smax(bounds->r64)); - if (bounds->r32.size < U32_MAX) { + if (bounds->r32.size < U32_MAX && + (cnum32_smin(bounds->r32) != cnum64_smin(bounds->r64) || + cnum32_smax(bounds->r32) != cnum64_smax(bounds->r64))) { bpf_log(log, " (s32 "); - bpf_verbose_snum(env, cnum32_smin(bounds->r32)); + bpf_verbose_snum32(env, cnum32_smin(bounds->r32)); bpf_log(log, ".."); - bpf_verbose_snum(env, cnum32_smax(bounds->r32)); + bpf_verbose_snum32(env, cnum32_smax(bounds->r32)); bpf_log(log, ")"); } bpf_log(log, " step %u", bounds->step); diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 9074b2f043d4..2103c58cfaad 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -1771,7 +1771,7 @@ __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("loop header at 1, header_count is 4") -__msg("loop header at 1, widening r0 to 0..3 (s32 0..3) step 1") +__msg("loop header at 1, widening r0 to 0..3 step 1") __msg("loop header at 1, clamping r0") __msg("4: R0=3") __naked void pre_cond_je32(void) From 88b1da8583260ec8cd157269dc9c2213b28c62bf Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:27 -0700 Subject: [PATCH 22/48] selftests/bpf: Pointer operands for 32-bit recurrences/latches Add four tests for pointer operands in 32-bit SCEV analysis. Check that loop bounds are not inferred for the following program shapes: 32-bit latch | 32-bit recurrence -------------------------+-------------------------- r6 = ... pointer ... | r6 = ... pointer ... r7 = ... pointer ... | r7 = ... pointer ... l: r6 += 1 | l: w6 += 1 ... | ... if w6 < w7 goto l | if r6 < r7 goto l Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-7-f373aec9825f@gmail.com --- .../selftests/bpf/progs/verifier_scev.c | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 2103c58cfaad..805157cfea78 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -2971,6 +2971,115 @@ __naked void ptr_stack_other_frame(void) " ::: __clobber_all); } +SEC("xdp") +__success +__log_level(2) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__msg("loop header at 11, can't compute iterations count at 14") +__naked void ptr_map_value_jmp32(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" r6 += 1; \ + if w6 < w7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__failure +__log_level(2) +__msg("loop header at 11, can't compute iterations count") +__msg("R6 32-bit pointer arithmetic prohibited") +__naked void ptr_map_value_alu32_latch(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" w6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 11, can't compute iterations count") +__naked void ptr_map_value_alu32_latch2(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" r6 += 1; \ + w8 = w6; \ + if r8 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 8, can't widen r0, expr is (zext32 (+ r0 1))") +__naked void ptr_map_value_alu32_non_latch(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r1 = 0; \ +1: r1 += 1; \ + w0 = w0; \ + w0 += 1; \ + if r1 < 4 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + /* Metadata and packet data have different origins. */ SEC("xdp") __success From 39cd4e0915c4d9661ec1fdd472b515b28c98eb37 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:28 -0700 Subject: [PATCH 23/48] selftests/bpf: Cases with empty range error under old clamping logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 64-bit and 32-bit examples leading to an empty register range error when scev.c:bpf_clamp_scev_regs() intersected current register state with computed iteration range: r1 = 0; verification r3 = 0; path loop: r3 += 1; | <---------. if r3 != 5 goto +0; | r3 ∈ {5} | r1 += 1; | | if r1 != 5 goto loop; '-----------' r0 = 0; exit; - At header r3 ∈ {0,1,2,3,4}. - After `if r3 != 5 goto ...` continued with r3 ∈ {5}. - On a second iteration bpf_clamp_scev_regs() computed: r3 := {0,1,2,3,4} ∩ {5} = ∅ And reported an error: verifier bug: scev clamp produced empty range for r3 Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-8-f373aec9825f@gmail.com --- .../selftests/bpf/progs/verifier_scev.c | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 805157cfea78..4e58dd475b88 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -1604,6 +1604,61 @@ __naked void post_cond_jne_neg_step(void) " ::: __clobber_all); } +/* + * r1 and r3 are widened independently, so the r3 == 5 path reaches the back + * edge with r3 past the last value it takes at the header. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 5 ") +__msg("loop header at 2, widening r1 to 0..4 step 1") +__msg("loop header at 2, widening r3 to 0..4 step 1") +__msg("3: (55) if r3 != 0x5 goto pc+1 {{.*}}; R3=5") +__msg("loop header at 2, clamping r3 to 0..4 step 1") +__msg("from 6 to 2: safe") +__naked void clamp_dependent_overshoot(void) +{ + asm volatile (" \ + r1 = 0; \ + r3 = 0; \ +1: r3 += 1; \ + if r3 != 5 goto 2f; \ + r2 = 0; \ +2: r1 += 1; \ + if r1 != 5 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Same with a 32-bit dependent register, SCEV (zext32 (linear r3 1)) */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 5 ") +__msg("loop header at 2, widening r1 to 0..4 step 1") +__msg("loop header at 2, widening r3 to 0..4 step 1") +__msg("3: (56) if w3 != 0x5 goto pc+1 {{.*}}; R3=5") +__msg("loop header at 2, clamping r3 to 0..4 step 1") +__msg("from 6 to 2: safe") +__naked void clamp_dependent_overshoot32(void) +{ + asm volatile (" \ + r1 = 0; \ + w3 = 0; \ +1: w3 += 1; \ + if w3 != 5 goto 2f; \ + r2 = 0; \ +2: r1 += 1; \ + if r1 != 5 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + SEC("xdp") __success __log_level(2) From 344c14e8b3d45e3c643cec1ba6acd3e4691dfe66 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:29 -0700 Subject: [PATCH 24/48] selftests/bpf: Cases for SCEV interaction with infinite loop detection Add three test cases exercising state pruning after loop counter clamping. All turn a 32-bit stack spill into STACK_MISC during the first iteration. This prevents a state match at the outer loop header on the next iteration, while allowing states to match further inside the loop. All are falsely reported as infinite loop w/o changes in patch [1]. - clamp_callee_state_pruning(), a callee overwrites the upper half of the caller's stack slot. States then match at the callee's exit, where the current frame has no active loop. Verify that current_loop() finds the caller's active loop. - clamp_nested_loop_state_pruning(), the overwrite happens inside an inner loop. Exercise matching states when the inner loop is entered again on a subsequent outer iteration. - clamp_nested_unrolled_loop_state_pruning(), same as above, but the inner loop is not recognized by SCEV and is unrolled with a precise counter, so its entries are never marked terminating. [1] bpf: avoid false infinite loop reports after changes in SCEV clamping Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-9-f373aec9825f@gmail.com --- .../selftests/bpf/progs/verifier_scev.c | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 4e58dd475b88..39603c8bf083 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -2567,6 +2567,104 @@ __naked void no_widen_any_stack_origin(void) " ::: __clobber_all); } +__naked __noinline __used static int scrub_slot_callee(void) +{ + asm volatile (" \ + *(u32 *)(r1 + 4) = 1; \ + call %[bpf_get_prandom_u32]; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * The callee turns the caller's fp-8 into misc, so the states at the call site + * differ between the first and the second iteration, while the states inside + * the callee are identical. Verification fails with an infinite loop error if + * states.c:current_loop() does not search upper frames for active loops. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__msg("12: safe") +__naked void clamp_callee_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: r1 = r10; \ + r1 += -8; \ + call scrub_slot_callee; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + +/* + * The inner loop turns fp-8 from a spill into misc. On the next outer + * iteration, the states inside the inner loop match its first invocation. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 4, widening r7 to 0..2 step 1") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__naked void clamp_nested_loop_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: r7 = 0; \ +2: *(u32 *)(r10 - 4) = 1; \ + r7 += 1; \ + if r7 < 3 goto 2b; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + +/* + * Same as above, but the inner loop is not recognized by SCEV and is unrolled + * with a precise r7. The states at its header still match between the outer + * iterations and must not be reported as an infinite loop: the path between + * them crosses the back edge of the terminating outer loop. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 5, can't compute iterations count") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__naked void clamp_nested_unrolled_loop_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: *(u32 *)(r10 - 4) = 1; \ + r7 = 1; \ +2: r7 *= 2; \ + if r7 < 4 goto 2b; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + /* * r7 starts as 1 + 6*k and invariant r8 as 1 + 9*k (0 <= k <= 3). * The (any r7 r8) union must preserve base 1 and step gcd(6, 9) = 3. From a0137c789774bdc2eff9b1b4525741ed118eeba0 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:30 -0700 Subject: [PATCH 25/48] selftests/bpf: 32-bit copy of the counter compared at the latch When building SCEV expressions for the loop: w7 = w1; r7 = (zext32 r1) w1 = w7; r1 = (zext32 (zext32 r1)) w1 += 2; r1 = (zext32 (+ (zext32 (zext32 r1)) 2)) Computed SCEV expression for r1: (zext32 (linear r1 2)) Substituting this into r7's expression at the latch: (zext32 (zext32 (linear r1 2))) Without (zext32 (zext32 X)) -> (zext32 X) simplification rule the latch would not be recognized as linear and header_count won't be computed. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-10-f373aec9825f@gmail.com --- .../selftests/bpf/progs/verifier_scev.c | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 39603c8bf083..37434efd1490 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3710,4 +3710,31 @@ __naked void dominating_exit_bounds_count(void) " ::: __clobber_all); } +/* + * The counter is copied to a callee-saved register for the call and the copy is compared. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg(" scev at latch 5:") +__msg(" r7=(zext32 r1) / (zext32 (linear r1 2))") +__msg("loop header at 1, header_count is 100 ") +__msg("loop header at 1, widening r1 to 0..198 step 2") +__naked void post_cond_jlt32_copy(void) +{ + asm volatile (" \ + w1 = 0; \ +1: w7 = w1; \ + call %[bpf_get_prandom_u32]; \ + w1 = w7; \ + w1 += 2; \ + if w7 < 198 goto 1b; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; From 8fd732e5a64b5c584b1c8701e6a8bb98da830205 Mon Sep 17 00:00:00 2001 From: Eduard Zingerman Date: Mon, 26 Oct 2026 16:38:31 -0700 Subject: [PATCH 26/48] selftests/bpf: Restore stack pointer subrange pruning coverage After the clamping changes, r7 returns to the header with [-72, -16] instead of the narrower [-64, -16], so the test no longer checks subrange pruning. Assign an invariant fp-16 pointer to r7 in the loop, giving r7 an ANY SCEV that is not clamped and restoring strict-subrange coverage. Signed-off-by: Eduard Zingerman Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261026-scev-simpler-clamp-v2-11-f373aec9825f@gmail.com --- .../selftests/bpf/progs/verifier_stack_ptr.c | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index dc176f407c36..e6a9021b197b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -588,32 +588,37 @@ __naked void stack_check_size_512_with_may_goto(void) /* * Verify that old PTR_TO_STACK state is considered a super-set of - * new PTR_TO_STACK state when new variable range is a sub-range - * of the old range, e.g. old [-72, -16] vs new [-64, -16]. + * new PTR_TO_STACK state when the new offset is within the old range, + * e.g. old [-72, -16] vs new {-16}. */ SEC("socket") __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) /* - * r7 is widened to [-72, -16] at the loop header (insn 3), - * the loop body sees [-64, -8] after 'r7 += 8' + * r7 is widened to [-72, -16] at the loop header (insn 5), + * the loop body sees [-64, -8] after 'r7 += 8'. */ -__msg("loop header at 3, widening r7 to -72..-16 step 8") +__msg("loop header at 5, widening r7 to -72..-16 step 1") __msg("R7=fp(smin=smin32=-64,smax=smax32=-8") -/* back-edge state is clamped to the widened range and pruned at the header */ -__msg("loop header at 3, clamping r7 to -72..-16 step 8") -__msg("from 5 to 3: safe") -__msg("processed 9 insns") +/* 'r7 = r8' sets r7 to fp-16; its ANY SCEV is not clamped. */ +__msg("R7=fp-16 R8=fp-16 loop_stack=5") +__not_msg("clamping r7") +/* back-edge state is pruned because {-16} is within [-72, -16]. */ +__msg("from 8 to 5: safe") +__msg("processed 12 insns") __naked void stack_ptr_subrange_in_loop(void) { asm volatile (" \ r7 = r10; \ r7 += -72; \ + r8 = r10; \ + r8 += -16; \ r6 = 0; \ - r7 += 8; \ +1: r7 += 8; \ + r7 = r8; \ r6 += 1; \ - if r6 < 8 goto -3; \ + if r6 < 8 goto 1b; \ r0 = 0; \ exit; \ " ::: __clobber_all); From 815a0a2a40d6c5985a153041231731651bc4024b Mon Sep 17 00:00:00 2001 From: Mani Tofigh Date: Tue, 27 Oct 2026 13:03:42 -0400 Subject: [PATCH 27/48] bpf: Fix bpf_timer callback running on a freed map A bpf_timer in an inner map can fire after the map has been freed, and bpf_timer_cb() then reads the freed map: [ 50.709520] BUG: KASAN: slab-use-after-free in bpf_timer_cb+0x249/0x290 [ 50.709540] Read of size 4 at addr ff11000003bc1030 by task bpftimer/72 ... [ 50.710783] Freed by task 12: ... [ 50.710826] bpf_map_free+0x1a2/0x380 When user space closes the map, its timers are cancelled and their callbacks are cleared. Since commit 1bfbc267ec91 ("bpf: Enable bpf_timer and bpf_wq in any context"), bpf_timer_set_callback() and bpf_timer_start() run without a lock, so a program that found the timer just before the close can still use it afterwards: Task A Task B Task C ------ ------ ------ bpf_timer_start() finds the timer bpf_timer_set_callback() finds the timer close(map_fd) clears the callback cancels the timer sets the callback arms the timer deletes the map from the outer map map is freed the timer fires bpf_timer_cb() reads the freed map The timer is cancelled once more when it is finally freed, but that happens well after the inner map is freed, so a timer started this way can still fire on the freed map. Clear the timer's map pointer in bpf_async_cancel_and_free() before the map can be freed, and make the bpf_timer and bpf_wq callbacks return early when it is NULL. The map is not freed until callbacks that are already running have finished, so a callback either sees the map while it is still valid or sees NULL. The reproducer below calls the two helpers in a loop on two CPUs while a third CPU keeps creating an inner map with a timer, closing it and removing it from the outer map. This bug was identified and confirmed with the help of a deterministic hypervisor currently under development, but the non-deterministic C reproducer can typically trigger it in the 1-50s time range. With the fix, nothing is triggered after 600 seconds. Fixes: 1bfbc267ec91 ("bpf: Enable bpf_timer and bpf_wq in any context") Signed-off-by: Mani Tofigh Signed-off-by: Alexei Starovoitov Tested-by: Mahe Tardy Link: https://patch.msgid.link/20261027-bpf-timer-freed-map-v1-1-30402b0ee99b@columbia.edu --- kernel/bpf/helpers.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 648abe1ba63a..58e357302850 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1221,7 +1221,7 @@ static void bpf_async_refcount_put(struct bpf_async_cb *cb); static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) { struct bpf_hrtimer *t = container_of(hrtimer, struct bpf_hrtimer, timer); - struct bpf_map *map = t->cb.map; + struct bpf_map *map = READ_ONCE(t->cb.map); void *value = t->cb.value; bpf_callback_t callback_fn; void *key; @@ -1229,7 +1229,7 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) BTF_TYPE_EMIT(struct bpf_timer); callback_fn = rcu_dereference_check(t->cb.callback_fn, rcu_read_lock_bh_held()); - if (!callback_fn) + if (!callback_fn || !map) goto out; /* bpf_timer_cb() runs in hrtimer_run_softirq. It doesn't migrate and @@ -1254,7 +1254,7 @@ static void bpf_wq_work(struct work_struct *work) { struct bpf_work *w = container_of(work, struct bpf_work, work); struct bpf_async_cb *cb = &w->cb; - struct bpf_map *map = cb->map; + struct bpf_map *map; bpf_callback_t callback_fn; void *value = cb->value; void *key; @@ -1265,8 +1265,9 @@ static void bpf_wq_work(struct work_struct *work) /* keeps the program that callback_fn belongs to */ rcu_read_lock_trace(); + map = READ_ONCE(cb->map); callback_fn = READ_ONCE(cb->callback_fn); - if (!callback_fn) + if (!callback_fn || !map) goto out; key = map_key_from_value(map, value, &idx); @@ -1733,6 +1734,8 @@ static void bpf_async_cancel_and_free(struct bpf_async_kern *async) if (!cb) return; + /* Callbacks that run after this see NULL and return. */ + WRITE_ONCE(cb->map, NULL); bpf_async_update_prog_callback(cb, NULL, NULL); /* * No refcount_inc_not_zero(&cb->refcnt) here. Dropping the last From 3c13b2e5a40e2984b0e0addd83a33631938138a0 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:02:56 +0000 Subject: [PATCH 28/48] selftests/bpf: Remove tests for global data in arena The .arena.data section and the BPF_F_ARENA_SCALAR flag let a prog in Rust keep its global data in arena. The subsequent patches remove the two, because the compiler can do this work. First remove the tests for .arena.data. One of the tests is a prog in Rust. It has no replacement, because LLVM does not put the globals of Rust in arena at this time. No other selftest is in Rust. Thus also remove the makefile rules that build Rust. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-2-alexei.starovoitov@gmail.com --- tools/testing/selftests/bpf/Makefile | 12 +- .../testing/selftests/bpf/Makefile.buildvars | 25 -- tools/testing/selftests/bpf/Makefile.skel | 18 +- .../selftests/bpf/prog_tests/data_in_arena.c | 216 ------------------ .../selftests/bpf/progs/data_in_arena.c | 112 --------- .../selftests/bpf/progs/data_in_arena_decl.c | 37 --- .../bpf/progs/data_in_arena_extern.c | 20 -- .../selftests/bpf/progs/data_in_arena_fail.c | 20 -- .../selftests/bpf/progs/data_in_arena_nomap.c | 20 -- .../selftests/bpf/progs/data_in_arena_rust.rs | 73 ------ 10 files changed, 2 insertions(+), 551 deletions(-) delete mode 100644 tools/testing/selftests/bpf/prog_tests/data_in_arena.c delete mode 100644 tools/testing/selftests/bpf/progs/data_in_arena.c delete mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_decl.c delete mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_extern.c delete mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_fail.c delete mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_nomap.c delete mode 100644 tools/testing/selftests/bpf/progs/data_in_arena_rust.rs diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 1e4af62e8312..c99e0f623e10 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -423,16 +423,6 @@ $(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) endif -# #![no_std] looks for compiler_builtins too. Nothing of it is used. -ifneq ($(RUST_CORE),) -$(RUST_CORE): $(RUST_CORE_SRC) - $(call msg,RUSTC,,$@) - $(Q)mkdir -p $(@D) - +$(Q)$(RUSTC_BPF) -A warnings --edition 2024 --crate-name core --out-dir $(@D) $< - +$(Q)echo '#![feature(compiler_builtins)] #![compiler_builtins] #![no_std]' | \ - $(RUSTC_BPF) -A warnings --crate-name compiler_builtins --out-dir $(@D) - -endif - # Generated test list headers define gen_tests_hdr @@ -468,7 +458,7 @@ RUNNER_PREREQS := $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(BPFTOOL) \ $(VERIFY_SIG_HDR) $(PRIVATE_KEY) $(VERIFICATION_CERT) \ $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) \ prog_tests/tests.h map_tests/tests.h \ - $(RUNNER_OBJS) $(RUST_CORE) + $(RUNNER_OBJS) # Runtime fixtures for each test_progs flavor. RUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ diff --git a/tools/testing/selftests/bpf/Makefile.buildvars b/tools/testing/selftests/bpf/Makefile.buildvars index ff3476bc40a4..d2a0c0031b87 100644 --- a/tools/testing/selftests/bpf/Makefile.buildvars +++ b/tools/testing/selftests/bpf/Makefile.buildvars @@ -109,31 +109,6 @@ HOST_INCLUDE_DIR := $(INCLUDE_DIR) endif RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids -# Programs in Rust are built by upstream rustc. It has no prebuilt core for -# the bpf target, so it has to come with the source of core: -# rustup component add rust-src -# core is built as edition 2024, which it is since rustc 1.87. -# rustc emits LLVM bitcode and clang makes the object of it, so clang has to be -# 23 or newer and not older than LLVM of rustc. -# Otherwise RUST_CORE is empty and the tests are skipped. -RUSTC ?= rustc -RUST_CORE_SRC := $(wildcard $(shell $(RUSTC) --print sysroot 2>/dev/null)$\ - /lib/rustlib/src/rust/library/core/src/lib.rs) -ifneq ($(RUST_CORE_SRC),) -ifeq ($(shell { clang=$$(echo __clang_major__ | $(CLANG) -E -P -x c -) && \ - llvm=$$($(srctree)/scripts/rustc-llvm-version.sh $(RUSTC)) && \ - [ $$($(srctree)/scripts/rustc-version.sh $(RUSTC)) -ge 108700 ] && \ - [ $$clang -ge 23 ] && [ $$clang -ge $$((llvm / 10000)) ]; } \ - 2>/dev/null && echo y),y) -RUST_CORE := $(BUILD_DIR)/rust/libcore.rlib -endif -endif -# RUSTC_BOOTSTRAP=1 is to build core with a stable rustc, like the kernel does. -# panic=abort is a stop gap until panic=unwind is supported. -RUSTC_BPF = RUSTC_BOOTSTRAP=1 $(RUSTC) -O -C panic=abort --crate-type rlib \ - --target $(if $(IS_LITTLE_ENDIAN),bpfel,bpfeb)-unknown-none \ - -L $(dir $(RUST_CORE)) - DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool ifneq ($(CROSS_COMPILE),) CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 6570c48f92b0..e8699903feef 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -20,8 +20,7 @@ ifneq ($(BPF_CC),) BPF_SRCS := $(notdir $(wildcard progs/*.c)) BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) -SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c \ - data_in_arena_extern.c data_in_arena_nomap.c +SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ linked_vars.skel.h linked_maps.skel.h linked_arena.skel.h \ @@ -138,21 +137,6 @@ $(LINKED_SKELS_H): $(RDIR)/%.skel.h: $$(addprefix $(RDIR)/,$$($$*.skel.h-deps)) $(BPFTOOL) $(GEN_SKEL) | $(RDIR) $(Q)$(cmd_bpf_link_skel) -# Programs in Rust, see Makefile.buildvars. No skeletons: the objects may be absent. -ifneq ($(RUST_CORE),) -ifeq ($(BPF_CC),$(CLANG)) -RUST_OBJS := $(patsubst progs/%.rs,$(RDIR)/%.bpf.o,$(wildcard progs/*.rs)) -BPF_OBJS += $(RUST_OBJS) - -$(RUST_OBJS): $(RDIR)/%.bpf.o: progs/%.rs $(RUST_CORE) | $(RDIR) - $(call msg,RUSTC,$(BINARY),$@) - +$(Q)$(RUSTC_BPF) --edition 2021 -C debuginfo=2 --emit=llvm-bc \ - $(patsubst -mcpu=%,-C target-cpu=%,$(filter -mcpu=%,$(BPF_CC_FLAGS))) \ - -o $(dir $(RUST_CORE))$(BINARY)-$*.bc $< && \ - $(BPF_CC) $(BPF_CC_FLAGS) -c $(dir $(RUST_CORE))$(BINARY)-$*.bc -o $@ $(call skip_on_fail,BPF) -endif -endif - # Programs in C++. No skeletons: gcc doesn't build them. ifeq ($(BPF_CC),$(CLANG)) CXX_OBJS := $(patsubst progs/%.cpp,$(RDIR)/%.bpf.o,$(wildcard progs/*.cpp)) diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c deleted file mode 100644 index cb1023507c01..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c +++ /dev/null @@ -1,216 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include "data_in_arena.skel.h" -#include "data_in_arena_decl.skel.h" -#include "data_in_arena_fail.skel.h" - -static int run_prog(struct bpf_program *prog) -{ - LIBBPF_OPTS(bpf_test_run_opts, topts); - - if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &topts), "test_run")) - return -1; - return topts.retval; -} - -static void run(struct data_in_arena *skel, int counter) -{ - int i; - - ASSERT_EQ(run_prog(skel->progs.use_data), counter + 7 + 1 + 2, "retval"); - ASSERT_EQ(skel->bss->sum, counter + 7 + 1 + 2, "sum"); - ASSERT_EQ(skel->data->counter, counter + 1, "counter"); - ASSERT_EQ(skel->data->pair[1], 5, "pair[1]"); - for (i = 0; i < 4; i++) - ASSERT_EQ(skel->bss->table[i], 10 * (i + 1) + i, "table"); -} - -static void test_in_arena(void) -{ - struct bpf_map_info info = {}; - struct data_in_arena *skel; - __u32 len = sizeof(info); - struct bpf_map *arena; - size_t sz; - - skel = data_in_arena__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - - arena = bpf_object__find_map_by_name(skel->obj, "arena"); - if (!ASSERT_OK_PTR(arena, "arena")) - goto out; - ASSERT_EQ(bpf_map__type(arena), BPF_MAP_TYPE_ARENA, "arena type"); - ASSERT_EQ(bpf_map__max_entries(arena), 1, "arena pages"); - ASSERT_OK(bpf_map__set_max_entries(arena, 8), "arena resize"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.data), "data autocreate"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.bss), "bss autocreate"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.rodata), "rodata autocreate"); - ASSERT_EQ(bpf_map__set_autocreate(skel->maps.data, true), -EOPNOTSUPP, "set_autocreate"); - ASSERT_EQ(bpf_map__set_value_size(skel->maps.bss, 4096), -EOPNOTSUPP, "set_value_size"); - ASSERT_EQ(bpf_map__initial_value(skel->maps.data, &sz), skel->data, "initial_value"); - ASSERT_EQ(sz, sizeof(*skel->data), "initial_value size"); - - /* initial values are set the usual way */ - skel->data->counter = 100; - - if (!ASSERT_OK(data_in_arena__load(skel), "load")) - goto out; - /* there are no maps behind the sections */ - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.data), &info, &len), "data map"); - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &info, &len), "bss map"); - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.rodata), &info, &len), - "rodata map"); - run(skel, 100); - - /* pointers to data next to pointers to functions */ - ASSERT_EQ(run_prog(skel->progs.use_ops), 42 + 1 + 'e', "use_ops"); - ASSERT_EQ(skel->data->counter, 102, "counter"); - - /* alignment of sections and pointers to data in data */ - ASSERT_EQ((unsigned long)&skel->bss->aligned64 % 64, 0, "alignment"); - ASSERT_EQ(run_prog(skel->progs.use_ptrs), 0, "use_ptrs"); - ASSERT_EQ(skel->data->x, 43, "x"); - ASSERT_EQ(skel->bss->aligned64.v[7], 7, "aligned64"); - ASSERT_EQ(skel->data->px, &skel->data->x, "px"); - - /* format strings of bpf_printk() and BPF_SNPRINTF() */ - ASSERT_EQ(run_prog(skel->progs.use_printk), sizeof("43-7"), "use_printk"); - ASSERT_STREQ(skel->bss->out, "43-7", "out"); -out: - data_in_arena__destroy(skel); -} - -/* The object has an arena map and __arena variables */ -static void test_declared_arena(void) -{ - struct data_in_arena_decl *skel; - struct bpf_map *map; - int arenas = 0; - - skel = data_in_arena_decl__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - bpf_object__for_each_map(map, skel->obj) - arenas += bpf_map__type(map) == BPF_MAP_TYPE_ARENA; - ASSERT_EQ(arenas, 1, "no second arena"); - skel->data->counter = 6; - if (!ASSERT_OK(data_in_arena_decl__load(skel), "load")) - goto out; - ASSERT_EQ(run_prog(skel->progs.use_data), 6 + 7 + 11, "retval"); - ASSERT_EQ(run_prog(skel->progs.use_data), 7 + 7 + 12, "retval"); - ASSERT_EQ(skel->bss->sum, 7 + 7 + 12, "sum"); - ASSERT_EQ(skel->data->counter, 8, "counter"); -out: - data_in_arena_decl__destroy(skel); -} - -/* A pointer in data that can't be made an address of arena fails the load */ -static void test_ptr_to_map(void) -{ - struct data_in_arena_fail *skel; - - skel = data_in_arena_fail__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - ASSERT_ERR(data_in_arena_fail__load(skel), "load"); - data_in_arena_fail__destroy(skel); -} - -/* So does a pointer to a variable of the kernel. There is no skeleton: the open fails. */ -static void test_ptr_to_extern(void) -{ - struct bpf_object *obj; - - obj = bpf_object__open_file("./data_in_arena_extern.bpf.o", NULL); - if (!ASSERT_ERR_PTR(obj, "open")) - bpf_object__close(obj); -} - -/* __arena variables and no arena map. There is no skeleton: the open fails. */ -static void test_arena_var_no_map(void) -{ - struct bpf_object *obj; - - obj = bpf_object__open_file("./data_in_arena_nomap.bpf.o", NULL); - if (!ASSERT_ERR_PTR(obj, "open")) - bpf_object__close(obj); -} - -/* The address of an arena that libbpf doesn't create is not known. No pointers in data then. */ -static void test_not_my_arena(bool pin) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); - struct data_in_arena *skel; - struct bpf_map *arena; - int fd = -1; - - skel = data_in_arena__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - arena = bpf_object__find_map_by_name(skel->obj, "arena"); - if (!ASSERT_OK_PTR(arena, "arena")) - goto out; - if (pin) { - ASSERT_OK(bpf_map__set_pin_path(arena, "/sys/fs/bpf/data_in_arena"), "pin_path"); - } else { - fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, 1, &opts); - if (!ASSERT_GE(fd, 0, "map_create")) - goto out; - ASSERT_OK(bpf_map__reuse_fd(arena, fd), "reuse_fd"); - } - ASSERT_EQ(data_in_arena__load(skel), -ENOTSUP, "load"); -out: - if (fd >= 0) - close(fd); - data_in_arena__destroy(skel); -} - -/* The object is there if rustc and clang can build it, see Makefile.buildvars */ -static void test_rust(void) -{ - const char *file = "./data_in_arena_rust.bpf.o"; - struct bpf_object *obj; - - if (access(file, R_OK)) { - test__skip(); - return; - } - obj = bpf_object__open_file(file, NULL); - if (!ASSERT_OK_PTR(obj, "open")) - return; - if (!ASSERT_OK(bpf_object__load(obj), "load")) - goto out; - /* libbpf goes on without BTF when the kernel doesn't take it */ - ASSERT_GE(bpf_object__btf_fd(obj), 0, "btf_fd"); - ASSERT_EQ(run_prog(bpf_object__find_program_by_name(obj, "list_in_data")), - 100 + 20 + 3, "retval"); -out: - bpf_object__close(obj); -} - -void test_data_in_arena(void) -{ -#if !defined(__x86_64__) && !defined(__aarch64__) - /* other JITs don't take BPF_F_ARENA_SCALAR */ - test__skip(); - return; -#endif - if (test__start_subtest("arena")) - test_in_arena(); - if (test__start_subtest("declared_arena")) - test_declared_arena(); - if (test__start_subtest("ptr_to_map")) - test_ptr_to_map(); - if (test__start_subtest("ptr_to_extern")) - test_ptr_to_extern(); - if (test__start_subtest("arena_var_no_map")) - test_arena_var_no_map(); - if (test__start_subtest("pinned_arena")) - test_not_my_arena(true); - if (test__start_subtest("reused_arena")) - test_not_my_arena(false); - if (test__start_subtest("rust")) - test_rust(); -} diff --git a/tools/testing/selftests/bpf/progs/data_in_arena.c b/tools/testing/selftests/bpf/progs/data_in_arena.c deleted file mode 100644 index 18f38103c0fc..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena.c +++ /dev/null @@ -1,112 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -long pair[2] = { 1, 2 }; -int x = 42; -long sum; -long table[4]; -struct { - long v[8]; -} aligned64 __attribute__((aligned(64))); -const volatile int ro = 7; -const volatile long ro_table[4] = { 10, 20, 30, 40 }; - -/* const strings stay in a map for helpers and kfuncs, a copy of them is in arena */ -const char hello[] SEC(".rodata.str.hello") = "hello"; - -/* pointers to data that are stored in data */ -int *px = &x; -const char *str = hello; -int *const volatile cpx SEC(".data.rel.ro") = &x; - -SEC("syscall") -int use_data(void *ctx) -{ - int i; - - for (i = 0; i < 4; i++) - table[i] = ro_table[i] + i; - sum = counter + ro + pair[0] + pair[1]; - counter++; - __sync_fetch_and_add(&pair[1], 3); - return sum; -} - -typedef int (*op_fn)(int); - -static __noinline int add1(int v) -{ - return v + 1; -} - -/* - * Pointers to functions and to data in read-only data of a program with callx. - * Volatile, so that the compiler doesn't replace the pointers with what - * they point to. - */ -static const volatile struct { - op_fn fn; - int *data; - const char *name; -} ops SEC(".data.rel.ro") = { add1, &x, hello }; - -SEC("syscall") -int use_ops(void *ctx) -{ - /* a program has the arena when its code refers to it */ - counter++; -#ifdef __clang__ - return ops.fn(*ops.data) + ops.name[1]; -#else - /* gcc doesn't support indirect calls */ - return add1(*ops.data) + ops.name[1]; -#endif -} - -SEC("syscall") -int use_ptrs(void *ctx) -{ - unsigned long addr = (unsigned long)&aligned64; - char local[4] = "abc"; - - /* hide the address from the compiler, it knows that '& 63' is 0 */ - asm volatile ("" : "+r"(addr)); - if (addr & 63) - return 1; - aligned64.v[7] = 7; - if (*px != 42) - return 2; - *px = 43; - if (x != 43 || *cpx != 43) - return 3; - if (str[0] != 'h' || str[4] != 'o' || str[5]) - return 4; - /* the literal is in a map */ - if (bpf_strncmp(local, sizeof(local), "abc")) - return 5; - return 0; -} - -char out[16]; - -/* format strings are in a map */ -SEC("syscall") -int use_printk(void *ctx) -{ - char buf[sizeof(out)]; - int i, n; - - bpf_printk("counter %d", counter); - n = BPF_SNPRINTF(buf, sizeof(buf), "%d-%d", x, ro); - for (i = 0; i < sizeof(out); i++) - out[i] = buf[i]; - return n; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c deleted file mode 100644 index 01bc4fea8f0c..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#define BPF_NO_KFUNC_PROTOTYPES -#include -#include -#include "bpf_experimental.h" -#include - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 4); -} arena SEC(".maps"); - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -long sum; -const volatile int ro = 7; - -#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) -int __arena avar = 11; -#else -int avar = 11; -#endif - -SEC("syscall") -int use_data(void *ctx) -{ - sum = counter + ro + avar; - counter++; - avar++; - return sum; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c deleted file mode 100644 index dc1ad5ca3bdd..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -extern const int bpf_prog_active __ksym; - -/* the variable of the kernel is not in arena */ -const void *kp = &bpf_prog_active; - -SEC("syscall") -int ptr_to_extern(void *ctx) -{ - return *(int *)kp; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c deleted file mode 100644 index ad8afe9afc96..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int x = 42; -/* the table is read-only data with pointers. It's not in arena. */ -int *const volatile tbl[1] SEC(".data.rel.ro") = { &x }; -int *const volatile *pp = tbl; - -SEC("syscall") -int ptr_to_map(void *ctx) -{ - return **pp; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c deleted file mode 100644 index ccf66321eb36..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "bpf_arena_common.h" - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -/* needs an arena map that is declared. The one that libbpf creates won't do. */ -int __arena_global avar = 11; - -SEC("syscall") -int arena_var(void *ctx) -{ - return avar + counter; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs deleted file mode 100644 index 3dd9d4208232..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -// Why .data, .bss and .rodata of a program in Rust are in arena. -// -// A reference in Rust is an address. It doesn't say what it points to and it -// can be stored in data. The list below has a node in each of the sections. -// The nodes are linked by references that are in the data: -// IN_BSS.next is stored by the program, -// IN_DATA.next is a relocation in .data against .rodata. -// sum() loads the references back and reads the three nodes with the same insn. -// -// When the sections are array maps libbpf skips the relocation in .data, and -// what sum() loads from a node is a number that can't be dereferenced: -// R1 invalid mem access 'scalar' -// In arena the address of a node is a number to begin with. - -#![no_std] -#![no_main] - -// Tell libbpf to keep .data, .bss and .rodata in arena. -#[used] -#[link_section = ".arena.data"] -static DATA_IN_ARENA: u8 = 0; - -#[used] -#[link_section = "license"] -static LICENSE: [u8; 4] = *b"GPL\0"; - -// panic=abort is a stop gap until panic=unwind is supported. -// Nothing here panics, so the handler is not a part of the program. -#[panic_handler] -fn panic(_info: &core::panic::PanicInfo) -> ! { - loop {} -} - -pub struct Node { - val: u32, - next: Option<&'static Node>, -} - -// no_mangle makes them visible outside, so LLVM can't fold the list into a constant. -#[no_mangle] -static IN_RODATA: Node = Node { val: 3, next: None }; -#[no_mangle] -static mut IN_DATA: Node = Node { - val: 20, - next: Some(&IN_RODATA), -}; -#[no_mangle] -static mut IN_BSS: Node = Node { val: 0, next: None }; - -#[inline(never)] -fn sum(mut node: Option<&Node>) -> u32 { - let mut sum = 0; - // The verifier wants a bound. - for _ in 0..8 { - let Some(n) = node else { break }; - sum += n.val; - node = n.next; - } - sum -} - -#[no_mangle] -#[link_section = "syscall"] -pub extern "C" fn list_in_data(_ctx: *mut u8) -> u32 { - unsafe { - let head = &mut *&raw mut IN_BSS; - head.val = 100; - head.next = Some(&*&raw const IN_DATA); - sum(Some(head)) - } -} From 5b0678b2f9b96fa44c9a47655428fe1706fdf9a7 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:02:57 +0000 Subject: [PATCH 29/48] libbpf: Remove .arena.data Rust has no address spaces. Thus a global variable cannot have the __arena attribute. The .arena.data section was a workaround. It told libbpf to do two things: - Move .data, .bss and .rodata into arena. - Load the progs with BPF_F_ARENA_SCALAR. The compiler must select the location of a variable, not libbpf. libbpf already puts the .addr_space.1 section in arena. The __arena variables of C are in this section, and LLVM can use it for Rust too. Until LLVM does that, the global data of Rust progs stays in array maps. This reverts commit 8ab58ed42b08 ("libbpf: Keep global data in arena when the object has .arena.data"). libbpf 1.8.0 does not have the section. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-3-alexei.starovoitov@gmail.com --- tools/lib/bpf/libbpf.c | 433 ++--------------------------------------- 1 file changed, 20 insertions(+), 413 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 8fcb8cf39cc0..fb8a1f07c5b6 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -552,26 +552,6 @@ struct bpf_struct_ops { #define STRUCT_OPS_SEC ".struct_ops" #define STRUCT_OPS_LINK_SEC ".struct_ops.link" #define ARENA_SEC ".addr_space.1" -/* - * An object with this section keeps its global data in arena instead of - * array maps. What is in the section doesn't matter. .data, .bss and .rodata - * sections become a part of the arena map of the object, like __arena global - * variables are. If the object doesn't declare an arena map libbpf creates - * one that is just large enough for the data. bpf_object__find_map_by_name() - * finds it by the name "arena" and bpf_map__set_max_entries() changes its - * size before the object is loaded. - * Read-only sections with pointers to functions and sections with constant - * strings stay frozen array maps. A constant that a helper or a kfunc takes - * by pointer has to be in such section. - * Pointers to data that are stored in data become addresses of arena. - * The load fails if such pointer points to a section that is not in arena, - * or if the arena is pinned or reused. - * - * It's for programs written in Rust. There are no address spaces in Rust, - * the program accesses arena through plain numbers, so programs of the object - * are loaded with BPF_F_ARENA_SCALAR. - */ -#define ARENA_DATA_SEC ".arena.data" enum libbpf_map_type { LIBBPF_MAP_UNSPEC, @@ -627,13 +607,6 @@ struct bpf_map { /* pointers to functions in the data of an internal map, see obj->func_ptrs */ struct func_ptr *func_ptrs; size_t func_ptr_cnt; - /* - * Data of the internal map is a part of arena data at arena_off. - * The map is not created, unless it's a copy that is in arena. - */ - bool in_arena; - bool arena_copy; - size_t arena_off; }; enum extern_type { @@ -808,17 +781,6 @@ struct bpf_object { void *arena_data; size_t arena_data_sz; size_t arena_data_off; - bool data_in_arena; - bool arena_mapped; - /* pointers to data in the data of internal maps, when data is in arena */ - struct data_ptr { - int sec_idx; - size_t sec_off; - int targ_sec_idx; - size_t targ_off; - char *sym_name; - } *data_ptrs; - size_t data_ptr_cnt; void *jumptables_data; size_t jumptables_data_sz; @@ -1644,7 +1606,6 @@ static struct bpf_object *bpf_object__new(const char *path, obj->efile.obj_buf = obj_buf; obj->efile.obj_buf_sz = obj_buf_sz; obj->efile.btf_maps_shndx = -1; - obj->efile.arena_data_shndx = -1; obj->kconfig_map_idx = -1; obj->arena_map_idx = -1; @@ -3126,136 +3087,6 @@ static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, return 0; } -static bool map_is_const_str(const struct bpf_map *map) -{ - return map->libbpf_type == LIBBPF_MAP_RODATA && - str_has_pfx(map->real_name, RODATA_SEC ".str"); -} - -static bool map_data_goes_to_arena(const struct bpf_object *obj, const struct bpf_map *map) -{ - int i; - - switch (map->libbpf_type) { - case LIBBPF_MAP_DATA: - case LIBBPF_MAP_BSS: - return true; - case LIBBPF_MAP_RODATA: - break; - default: - return false; - } - - /* - * Const strings that kfuncs and helpers take have to be in a frozen - * map. Pointers to them that are stored in data point to a copy of - * the strings that is in arena. - */ - if (map_is_const_str(map)) - return true; - /* - * Read-only data with pointers stays a frozen map: the kernel - * recognizes pointers to functions in it. - */ - if (str_has_pfx(map->real_name, DATA_REL_RO_SEC)) - return false; - for (i = 0; i < obj->efile.sec_cnt; i++) { - const struct elf_sec_desc *sec = &obj->efile.secs[i]; - - if (sec->sec_type == SEC_RELO && sec->shdr->sh_info == map->sec_idx) - return false; - } - return true; -} - -/* Returns the offset of the data of the map in arena data that is sz bytes so far */ -static size_t map_arena_off(const struct bpf_object *obj, const struct bpf_map *map, size_t sz) -{ - size_t align = obj->efile.secs[map->sec_idx].shdr->sh_addralign; - - return roundup(sz, max(align, sizeof(__u64))); -} - -/* - * Make .data, .bss and .rodata a part of arena data. They follow __arena - * variables of the object, if there are any. - */ -static int bpf_object__init_arena_data(struct bpf_object *obj) -{ - const size_t page_sz = sysconf(_SC_PAGE_SIZE); - size_t sz = obj->arena_data_sz; - struct bpf_map *map; - void *data; - int i; - - if (!obj->data_in_arena) - return 0; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - if (map_data_goes_to_arena(obj, map)) - sz = map_arena_off(obj, map, sz) + map->def.value_size; - } - if (sz == obj->arena_data_sz) - return 0; - - if (obj->arena_map_idx < 0) { - /* bpf_object__init_user_btf_maps() didn't look: there is no .maps section */ - if (obj->efile.arena_data) { - pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n", - ARENA_SEC); - return -ENOENT; - } - - map = bpf_object__add_map(obj); - if (IS_ERR(map)) - return PTR_ERR(map); - - map->real_name = strdup("arena"); - map->name = strdup("arena"); - if (!map->real_name || !map->name) { - zfree(&map->real_name); - zfree(&map->name); - return -ENOMEM; - } - map->sec_idx = -1; - map->def.type = BPF_MAP_TYPE_ARENA; - map->def.max_entries = roundup(sz, page_sz) / page_sz; - map->def.map_flags = BPF_F_MMAPABLE; - obj->arena_map_idx = map - obj->maps; - } - - data = realloc(obj->arena_data, sz); - if (!data) - return -ENOMEM; - memset(data + obj->arena_data_sz, 0, sz - obj->arena_data_sz); - sz = obj->arena_data_sz; - obj->arena_data = data; - obj->maps[obj->arena_map_idx].mmaped = data; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - if (!map_data_goes_to_arena(obj, map)) - continue; - sz = map_arena_off(obj, map, sz); - map->arena_off = sz; - memcpy(data + sz, map->mmaped, map->def.value_size); - sz += map->def.value_size; - pr_debug("map '%s': data is in arena at offset %zu\n", map->name, map->arena_off); - if (map_is_const_str(map)) { - map->arena_copy = true; - continue; - } - munmap(map->mmaped, bpf_map_mmap_sz(map)); - /* make bpf_map__initial_value() and bpf_map__set_initial_value() work */ - map->mmaped = data + map->arena_off; - map->autocreate = false; - map->in_arena = true; - } - obj->arena_data_sz = sz; - return 0; -} - static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, const char *pin_root_path) { @@ -3348,7 +3179,6 @@ static int bpf_object__init_maps(struct bpf_object *obj, err = err ?: bpf_object__init_global_data_maps(obj); err = err ?: bpf_object__init_kconfig_map(obj); err = err ?: bpf_object_init_struct_ops(obj); - err = err ?: bpf_object__init_arena_data(obj); return err; } @@ -4163,11 +3993,6 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!sh) return -LIBBPF_ERRNO__FORMAT; - /* the second pass has to know it when it sees relocations of data */ - name = elf_sec_str(obj, sh->sh_name); - if (name && strcmp(name, ARENA_DATA_SEC) == 0) - obj->data_in_arena = true; - if (sh->sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { pr_warn("elf: multiple symbol tables in %s\n", obj->path); @@ -4284,8 +4109,6 @@ static int bpf_object__elf_collect(struct bpf_object *obj) } else if (strcmp(name, ARENA_SEC) == 0) { obj->efile.arena_data = data; obj->efile.arena_data_shndx = idx; - } else if (strcmp(name, ARENA_DATA_SEC) == 0) { - /* the marker, see the first pass */ } else if (strcmp(name, JUMPTABLES_SEC) == 0) { obj->jumptables_data = malloc(data->d_size); if (!obj->jumptables_data) @@ -4310,9 +4133,6 @@ static int bpf_object__elf_collect(struct bpf_object *obj) * have pointers to functions. */ if (!section_have_execinstr(obj, targ_sec_idx) && - !(obj->data_in_arena && - (!strcmp(name, ".rel" DATA_SEC) || - str_has_pfx(name, ".rel" DATA_SEC "."))) && strcmp(name, ".rel" RODATA_SEC) && !str_has_pfx(name, ".rel" RODATA_SEC ".") && strcmp(name, ".rel" DATA_REL_RO_SEC) && @@ -5064,10 +4884,6 @@ static int bpf_program__record_reloc(struct bpf_program *prog, reloc_desc->insn_idx = insn_idx; reloc_desc->map_idx = map_idx; reloc_desc->sym_off = sym->st_value; - if (map->in_arena) { - reloc_desc->map_idx = obj->arena_map_idx; - reloc_desc->sym_off += map->arena_off; - } return 0; } @@ -5286,9 +5102,6 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) if (map_is_created(map)) return libbpf_err(-EBUSY); - if (map->in_arena) - return libbpf_err(-EOPNOTSUPP); - map->autocreate = autocreate; return 0; } @@ -5569,45 +5382,6 @@ bpf_object__reuse_map(struct bpf_map *map) return 0; } -static struct bpf_map *bpf_object__sec_map(struct bpf_object *obj, int sec_idx) -{ - int i; - - for (i = 0; i < obj->nr_maps; i++) - if (bpf_map__is_internal(&obj->maps[i]) && obj->maps[i].sec_idx == sec_idx) - return &obj->maps[i]; - return NULL; -} - -static int bpf_object__mmap_arena(struct bpf_object *obj, struct bpf_map *map) -{ - int i, err; - - map->mmaped = mmap((void *)(long)map->map_extra, - bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, - map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, - map->fd, 0); - if (map->mmaped == MAP_FAILED) { - err = -errno; - map->mmaped = NULL; - pr_warn("map '%s': failed to mmap arena: %s\n", - map->name, errstr(err)); - return err; - } - if (obj->arena_data) { - memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, - obj->arena_data_sz); - zfree(&obj->arena_data); - } - for (i = 0; i < obj->nr_maps; i++) { - struct bpf_map *m = &obj->maps[i]; - - if (m->in_arena) - m->mmaped = map->mmaped + obj->arena_data_off + m->arena_off; - } - return 0; -} - static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { @@ -5968,10 +5742,6 @@ bpf_object__create_maps(struct bpf_object *obj) continue; } - /* see bpf_object__relocate_data_ptrs() */ - if (map->def.type == BPF_MAP_TYPE_ARENA && obj->arena_mapped) - continue; - err = map_set_def_max_entries(map); if (err) goto err_out; @@ -6009,9 +5779,22 @@ bpf_object__create_maps(struct bpf_object *obj) if (err < 0) goto err_out; } else if (map->def.type == BPF_MAP_TYPE_ARENA) { - err = bpf_object__mmap_arena(obj, map); - if (err) + map->mmaped = mmap((void *)(long)map->map_extra, + bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, + map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, + map->fd, 0); + if (map->mmaped == MAP_FAILED) { + err = -errno; + map->mmaped = NULL; + pr_warn("map '%s': failed to mmap arena: %s\n", + map->name, errstr(err)); return err; + } + if (obj->arena_data) { + memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, + obj->arena_data_sz); + zfree(&obj->arena_data); + } } if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { err = init_map_in_map_slots(obj, map); @@ -8026,81 +7809,6 @@ static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_progra return err; } -/* - * Turn pointers to data that are stored in data into addresses of arena. - * The arena is created here, ahead of the other maps: the address has to be - * known before the maps that hold the pointers are created. - */ -static int bpf_object__relocate_data_ptrs(struct bpf_object *obj) -{ - struct bpf_map *arena, *map, *targ; - __u64 addr, val; - size_t i; - int err; - - if (!obj->data_ptr_cnt) - return 0; - - for (i = 0; i < obj->data_ptr_cnt; i++) { - struct data_ptr *p = &obj->data_ptrs[i]; - - if (p->targ_sec_idx < 0) - continue; - map = bpf_object__sec_map(obj, p->sec_idx); - targ = bpf_object__sec_map(obj, p->targ_sec_idx); - if (map && (map->in_arena || map->autocreate) && - (!targ || (!targ->in_arena && !targ->arena_copy))) { - pr_warn("sec '%s': pointer to '%s' at offset %zu can't be resolved: sec '%s' is not in arena\n", - map->real_name, p->sym_name, p->sec_off, - targ ? targ->real_name : ""); - return -LIBBPF_ERRNO__RELOC; - } - } - - if (obj->gen_loader) { - pr_warn("pointers to data in data are not supported by light skeleton\n"); - return -ENOTSUP; - } - - if (obj->arena_map_idx < 0) - return 0; - - arena = &obj->maps[obj->arena_map_idx]; - if (!arena->autocreate) - return 0; - /* libbpf doesn't mmap an arena that it didn't create, so the address is not known */ - if (arena->reused || arena->pin_path) { - pr_warn("map '%s': pointers to data in data are not supported with pinned or reused arena\n", - arena->name); - return -ENOTSUP; - } - - err = bpf_object__create_map(obj, arena, false); - err = err ?: bpf_object__mmap_arena(obj, arena); - if (err) { - pr_warn("map '%s': failed to create: %s\n", arena->name, errstr(err)); - return err; - } - obj->arena_mapped = true; - - for (i = 0; i < obj->data_ptr_cnt; i++) { - struct data_ptr *p = &obj->data_ptrs[i]; - - map = bpf_object__sec_map(obj, p->sec_idx); - if (!map || (!map->in_arena && !map->autocreate)) - continue; - - addr = (__u64)(unsigned long)arena->mmaped + obj->arena_data_off + p->targ_off; - if (p->targ_sec_idx >= 0) - addr += bpf_object__sec_map(obj, p->targ_sec_idx)->arena_off; - - memcpy(&val, map->mmaped + p->sec_off, sizeof(val)); - val += addr; - memcpy(map->mmaped + p->sec_off, &val, sizeof(val)); - } - return 0; -} - static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) { struct bpf_program *prog; @@ -8123,10 +7831,8 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat size_t mmap_sz = bpf_map_mmap_sz(arena_map); if (data_sz > mmap_sz) { - pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global %s of size %zu\n", - arena_map->name, mmap_sz, - obj->data_in_arena ? "data" : "__arena variables", - obj->arena_data_sz); + pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", + arena_map->name, mmap_sz, obj->arena_data_sz); return -E2BIG; } @@ -8135,10 +7841,6 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat obj->arena_data_off = mmap_sz - data_sz; } - err = bpf_object__relocate_data_ptrs(obj); - if (err) - return err; - /* Before relocating calls pre-process relocations and mark * few ld_imm64 instructions that points to subprogs. * Otherwise bpf_object__reloc_code() later would have to consider @@ -8371,85 +8073,12 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, return 0; } -/* - * A pointer to data that is stored in data. When data is in arena - * the program dereferences what it loads from data, so the pointer has - * to be the address of the target in arena. - */ -static int bpf_object__collect_data_ptr(struct bpf_object *obj, const char *relo_sec_name, - int relo_idx, size_t sec_idx, const Elf64_Rel *rel, - const Elf64_Sym *sym) -{ - Elf_Data *scn_data = obj->efile.secs[sec_idx].data; - const char *sym_name = elf_sym_str(obj, sym->st_name) ?: ""; - struct data_ptr *ptrs; - - if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_shndx < obj->efile.sec_cnt) - sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)) ?: ""; - - if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || - sym->st_shndx >= obj->efile.sec_cnt || - (sym->st_shndx != obj->efile.arena_data_shndx && - !bpf_object__shndx_is_data(obj, sym->st_shndx)) || - rel->r_offset >= scn_data->d_size || - scn_data->d_size - rel->r_offset < sizeof(__u64)) { - pr_warn("sec '%s': relo #%d: can't resolve pointer to '%s' at offset %zu when data is in arena\n", - relo_sec_name, relo_idx, sym_name, (size_t)rel->r_offset); - return -LIBBPF_ERRNO__RELOC; - } - - ptrs = libbpf_reallocarray(obj->data_ptrs, obj->data_ptr_cnt + 1, sizeof(*ptrs)); - if (!ptrs) - return -ENOMEM; - obj->data_ptrs = ptrs; - - ptrs[obj->data_ptr_cnt].sec_idx = sec_idx; - ptrs[obj->data_ptr_cnt].sec_off = rel->r_offset; - /* __arena variables are at the start of arena data */ - ptrs[obj->data_ptr_cnt].targ_sec_idx = - sym->st_shndx == obj->efile.arena_data_shndx ? -1 : sym->st_shndx; - ptrs[obj->data_ptr_cnt].targ_off = sym->st_value; - ptrs[obj->data_ptr_cnt].sym_name = strdup(sym_name); - if (!ptrs[obj->data_ptr_cnt].sym_name) - return -ENOMEM; - obj->data_ptr_cnt++; - - pr_debug("sec '%s': relo #%d: pointer at offset %zu to '%s'\n", - relo_sec_name, relo_idx, (size_t)rel->r_offset, sym_name); - return 0; -} - -/* Collect pointers in a data section that went to arena */ -static int bpf_object__collect_data_relos(struct bpf_object *obj, - Elf64_Shdr *shdr, Elf_Data *data) -{ - int i, err, nrels = shdr->sh_size / shdr->sh_entsize; - const char *relo_sec_name; - Elf64_Sym *sym; - Elf64_Rel *rel; - - relo_sec_name = elf_sec_str(obj, shdr->sh_name) ?: ""; - for (i = 0; i < nrels; i++) { - rel = elf_rel_by_idx(data, i); - sym = rel ? elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)) : NULL; - if (!sym) { - pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); - return -LIBBPF_ERRNO__FORMAT; - } - err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, shdr->sh_info, rel, sym); - if (err) - return err; - } - return 0; -} - /* * Collect pointers to functions in a read-only data section. They are * R_BPF_64_ABS64 relocations against .text section, where the offset of * a static function in the section is stored in place. Relocations in data * sections were ignored before pointers to functions were supported. Those - * that are something else, e.g. pointers to data, still are, unless data - * is in arena. + * that are something else, e.g. pointers to data, still are. */ static int bpf_object__collect_rodata_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) @@ -8485,15 +8114,6 @@ static int bpf_object__collect_rodata_relos(struct bpf_object *obj, if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || !sym_is_subprog(sym, obj->efile.text_shndx)) { - int err; - - if (obj->data_in_arena) { - err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, - sec_idx, rel, sym); - if (err) - return err; - continue; - } pr_debug("sec '%s': relo #%d: not a pointer to a function, skipping...\n", relo_sec_name, i); continue; @@ -8569,8 +8189,6 @@ static int bpf_object__collect_relos(struct bpf_object *obj) if (obj->efile.secs[idx].sec_type == SEC_RODATA) err = bpf_object__collect_rodata_relos(obj, shdr, data); - else if (obj->efile.secs[idx].sec_type == SEC_DATA) - err = bpf_object__collect_data_relos(obj, shdr, data); else if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) err = bpf_object__collect_st_ops_relos(obj, shdr, data); else if (idx == obj->efile.btf_maps_shndx) @@ -8914,9 +8532,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog } load_attr.log_level = log_level; load_attr.prog_flags = prog->prog_flags; - /* the program accesses its data in arena through plain numbers */ - if (obj->data_in_arena) - load_attr.prog_flags |= BPF_F_ARENA_SCALAR; load_attr.fd_array = obj->fd_array; err = bpf_object__bind_arena(obj, prog, &load_attr); if (err) @@ -8991,7 +8606,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog for (i = 0; i < obj->nr_maps; i++) { map = &prog->obj->maps[i]; - if (map->libbpf_type != LIBBPF_MAP_RODATA || map->in_arena) + if (map->libbpf_type != LIBBPF_MAP_RODATA) continue; if (bpf_prog_bind_map(ret, map->fd, NULL)) { @@ -10557,7 +10172,7 @@ static void bpf_map__destroy(struct bpf_map *map) zfree(&map->init_slots); map->init_slots_sz = 0; - if (map->mmaped && map->mmaped != map->obj->arena_data && !map->in_arena) + if (map->mmaped && map->mmaped != map->obj->arena_data) munmap(map->mmaped, bpf_map_mmap_sz(map)); map->mmaped = NULL; @@ -10642,9 +10257,6 @@ void bpf_object__close(struct bpf_object *obj) close(obj->func_ptr_maps[i].fd); zfree(&obj->func_ptr_maps); zfree(&obj->func_ptrs); - for (i = 0; i < obj->data_ptr_cnt; i++) - zfree(&obj->data_ptrs[i].sym_name); - zfree(&obj->data_ptrs); if (obj->btf_module_allowlist) { for (i = 0; i < obj->btf_module_allowlist_cnt; i++) @@ -11011,8 +10623,6 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; - if (obj->data_in_arena) - attr.prog_flags |= BPF_F_ARENA_SCALAR; prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); if (!prog_btf_fd && obj->btf) @@ -12019,9 +11629,6 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) if (map_is_created(map)) return libbpf_err(-EBUSY); - if (map->in_arena) - return libbpf_err(-EOPNOTSUPP); - if (map->mmaped) { size_t mmap_old_sz, mmap_new_sz; int err; From 58cbd31c399b37de1a885c2dee7e6d0e350ed32a Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:02:58 +0000 Subject: [PATCH 30/48] libbpf: Keep format strings of bpf_printk() in .rodata The format strings of bpf_printk() moved from .rodata to .rodata.str for one reason. .arena.data moved .rodata into arena, and helpers cannot read strings from arena. .arena.data is removed. Move the strings back to .rodata. As a result, an object that has its own .rodata has one map less. This reverts commit 373e1223ff00 ("libbpf: Keep format strings of bpf_printk() in .rodata.str"). libbpf 1.8.0 has the strings in .rodata. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-4-alexei.starovoitov@gmail.com --- tools/lib/bpf/bpf_helpers.h | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index b37b727ef216..9d160b5b9c0e 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h @@ -248,21 +248,13 @@ enum libbpf_tristate { #define ___bpf_fill(arr, args...) \ ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) -/* - * Format strings are in a section of their own. When global data of the object - * is in arena (the object has .arena.data section) libbpf keeps .rodata.str* - * sections in read-only maps, which is where helpers and kfuncs take strings - * from. - */ -#define ___bpf_fmt_sec __attribute__((section(".rodata.str"))) - /* * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values * in a structure. */ #define BPF_SEQ_PRINTF(seq, fmt, args...) \ ({ \ - static const char ___fmt[] ___bpf_fmt_sec = fmt; \ + static const char ___fmt[] = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -280,7 +272,7 @@ enum libbpf_tristate { */ #define BPF_SNPRINTF(out, out_size, fmt, args...) \ ({ \ - static const char ___fmt[] ___bpf_fmt_sec = fmt; \ + static const char ___fmt[] = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -295,7 +287,7 @@ enum libbpf_tristate { #ifdef BPF_NO_GLOBAL_DATA #define BPF_PRINTK_FMT_MOD #else -#define BPF_PRINTK_FMT_MOD static const ___bpf_fmt_sec +#define BPF_PRINTK_FMT_MOD static const #endif #define __bpf_printk(fmt, ...) \ @@ -311,7 +303,7 @@ enum libbpf_tristate { */ #define __bpf_vprintk(fmt, args...) \ ({ \ - static const char ___fmt[] ___bpf_fmt_sec = fmt; \ + static const char ___fmt[] = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -325,7 +317,7 @@ enum libbpf_tristate { #define bpf_stream_printk(stream_id, fmt, args...) \ ({ \ - static const char ___fmt[] ___bpf_fmt_sec = fmt; \ + static const char ___fmt[] = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ From fe8e06aa0575ded3200264b3ae1c56f7fb5e1347 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:02:59 +0000 Subject: [PATCH 31/48] selftests/bpf: Remove tests for arena access through numbers With BPF_F_ARENA_SCALAR the verifier accepts a load or a store through a scalar as an access to arena. The next patch removes the flag. Remove its tests. Keep one test. Without the flag the verifier must reject such a load, also in a prog that has an arena. Move this test to verifier_arena.c. The removed tests were the only tests of a special path in the x86 JIT for the fetching AND/OR/XOR atomics. Only the flag used that path. arena_atomics tests the usual path. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-5-alexei.starovoitov@gmail.com --- .../bpf/prog_tests/arena_scalar_blinded.c | 21 - .../selftests/bpf/prog_tests/verifier.c | 2 - .../selftests/bpf/progs/arena_noarena.c | 15 - .../selftests/bpf/progs/verifier_arena.c | 15 + .../bpf/progs/verifier_arena_scalar.c | 1155 ----------------- tools/testing/selftests/bpf/test_loader.c | 2 - 6 files changed, 15 insertions(+), 1195 deletions(-) delete mode 100644 tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c delete mode 100644 tools/testing/selftests/bpf/progs/verifier_arena_scalar.c diff --git a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c deleted file mode 100644 index 2ac2e9a652fe..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include "sysctl_helpers.h" -#include "verifier_arena_scalar.skel.h" - -/* The same tests with constants of the programs blinded */ -void serial_test_arena_scalar_blinded(void) -{ - const char *harden = "/proc/sys/net/core/bpf_jit_harden"; - char old[16] = {}; - - if (!is_jit_enabled()) { - test__skip(); - return; - } - if (sysctl_set_or_fail(harden, old, "2")) - return; - RUN_TESTS(verifier_arena_scalar); - sysctl_set_or_fail(harden, NULL, old); -} diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 04740ba37e91..6a494c381266 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -13,7 +13,6 @@ #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" -#include "verifier_arena_scalar.skel.h" #include "verifier_arena_globals1.skel.h" #include "verifier_arena_globals2.skel.h" #include "verifier_array_access.skel.h" @@ -207,7 +206,6 @@ void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } -void test_verifier_arena_scalar(void) { RUN(verifier_arena_scalar); } void test_verifier_arena_globals1(void) { RUN(verifier_arena_globals1); } void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } diff --git a/tools/testing/selftests/bpf/progs/arena_noarena.c b/tools/testing/selftests/bpf/progs/arena_noarena.c index edbd40cd5be8..1f46e5aa403b 100644 --- a/tools/testing/selftests/bpf/progs/arena_noarena.c +++ b/tools/testing/selftests/bpf/progs/arena_noarena.c @@ -21,19 +21,4 @@ int arena_arg_no_arena(void *ctx) return 0; } -SEC("syscall") -__flag(BPF_F_ARENA_SCALAR) -__arch_x86_64 -__arch_arm64 -__description("arena_scalar: no arena, no access through a number") -__failure __msg("R6 invalid mem access 'scalar'") -__naked void no_arena(void) -{ - asm volatile (" \ - r6 = 0x100000000000 ll; \ - r0 = *(u64 *)(r6 + 0); \ - exit; \ -" ::: __clobber_all); -} - char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 1ce9c4b78481..b17a6dfc100d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -610,6 +610,21 @@ int mixed_arena_scalar_alu64_arena_first(void *ctx) return 0; } +SEC("syscall") +__description("arena: a number is not an address in arena") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void access_through_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + SEC("syscall") __success __retval(0) int scalar_xor_arena_ptr(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c deleted file mode 100644 index 6f0efd9202ca..000000000000 --- a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c +++ /dev/null @@ -1,1155 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "../../../include/linux/filter.h" -#include "bpf_misc.h" - -void *bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, int node_id, - __u64 flags) __ksym; -void bpf_arena_free_pages(void *map, void *ptr, __u32 page_cnt) __ksym; - -#ifdef __TARGET_ARCH_arm64 -#define ARENA_VM_START (1ull << 32) -#else -#define ARENA_VM_START (1ull << 44) -#endif - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 4); - __ulong(map_extra, ARENA_VM_START); -} arena SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 1); - __type(key, int); - __type(value, long long); -} hash SEC(".maps"); - -/* JITs that take BPF_F_ARENA_SCALAR */ -#define __arena_scalar __flag(BPF_F_ARENA_SCALAR) __arch_x86_64 __arch_arm64 - -/* BTF FUNC records are not generated for kfuncs referenced from inline assembly */ -void __kfunc_btf_root(void) -{ - bpf_arena_alloc_pages(0, 0, 0, 0, 0); - bpf_arena_free_pages(0, 0, 0); -} - -/* Tests start with r6 = address of a new page as the user space sees it, a number */ - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load and store of every size") -__success __retval(0) -__load_if_JITed() -__naked void ld_st_sizes(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - r1 = 0x1122334455667788 ll; \ - *(u64 *)(r6 + 0) = r1; \ - *(u32 *)(r6 + 8) = r1; \ - *(u16 *)(r6 + 12) = r1; \ - *(u8 *)(r6 + 14) = r1; \ - *(u64 *)(r6 + 16) = 0x1234; \ - *(u32 *)(r6 + 24) = 0x5678; \ - *(u16 *)(r6 + 28) = 0x9a; \ - *(u8 *)(r6 + 30) = 0xbc; \ - r0 = 1; \ - r2 = *(u64 *)(r6 + 0); \ - if r2 != r1 goto 9f; \ - r0 = 2; \ - r2 = *(u32 *)(r6 + 8); \ - if r2 != 0x55667788 goto 9f; \ - r0 = 3; \ - r2 = *(u16 *)(r6 + 12); \ - if r2 != 0x7788 goto 9f; \ - r0 = 4; \ - r2 = *(u8 *)(r6 + 14); \ - if r2 != 0x88 goto 9f; \ - r0 = 5; \ - r2 = *(u64 *)(r6 + 16); \ - if r2 != 0x1234 goto 9f; \ - r0 = 6; \ - r2 = *(u32 *)(r6 + 24); \ - if r2 != 0x5678 goto 9f; \ - r0 = 7; \ - r2 = *(u16 *)(r6 + 28); \ - if r2 != 0x9a goto 9f; \ - r0 = 8; \ - r2 = *(u8 *)(r6 + 30); \ - if r2 != 0xbc goto 9f; \ - /* the address is what it was */ \ - r0 = 9; \ - if r6 != r7 goto 9f; \ - r0 = 10; \ - r7 >>= 32; \ - if r7 == 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load into the register that holds the address") -__success __retval(0) -__load_if_JITed() -__naked void ld_into_base(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 77; \ - *(u64 *)(r6 + 0) = r1; \ - r6 = *(u64 *)(r6 + 0); \ - r0 = 1; \ - if r6 != 77 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: sign extending load") -__success __retval(0) -__load_if_JITed() -__naked void ldsx(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - *(u64 *)(r6 + 0) = 0x80; \ - .8byte %[ldsx_insn]; /* r2 = *(s8 *)(r6 + 0) */ \ - r0 = 1; \ - if r2 != -128 goto 9f; \ - r0 = 2; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(ldsx_insn, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_B, - BPF_REG_2, BPF_REG_6, 0, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: address loaded from arena") -__success __retval(0) -__load_if_JITed() -__naked void ptr_chase(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - /* page[0] = &page[64]; page[64] = 5; */ \ - r1 = r6; \ - r1 += 64; \ - *(u64 *)(r6 + 0) = r1; \ - *(u64 *)(r1 + 0) = 5; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 1; \ - if r2 != r1 goto 9f; \ - r3 = *(u64 *)(r2 + 0); \ - r0 = 2; \ - if r3 != 5 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: atomics") -__success __retval(0) -__load_if_JITed() -__naked void atomics(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - *(u64 *)(r6 + 8) = 1; \ - r1 = 2; \ - lock *(u64 *)(r6 + 8) += r1; \ - r1 = 4; \ - .8byte %[fetch_add_insn]; /* r1 = atomic_fetch_add((u64 *)(r6 + 8), r1) */ \ - r0 = 1; \ - if r1 != 3 goto 9f; \ - r1 = 8; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r6 + 8, r1) */ \ - r0 = 2; \ - if r1 != 7 goto 9f; \ - r0 = 8; \ - r1 = 16; \ - .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r6 + 8, r0, r1) */ \ - r2 = r0; \ - r0 = 3; \ - if r2 != 8 goto 9f; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != 16 goto 9f; \ - r0 = 5; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching and/or/xor") -__success __retval(0) -__load_if_JITed() -__naked void fetch_and_or_xor(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = 0x11000000110 ll; \ - *(u64 *)(r6 + 8) = r7; \ - r0 = r6; \ - r0 ^= 0x55; \ - r1 = 0x1100000011 ll; \ - .8byte %[fetch_or_dw]; /* r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ - r2 = r6; \ - r2 ^= 0x55; \ - if r0 == r2 goto 1f; \ - r0 = 99; \ - goto 9f; \ -1: r0 = 1; \ - if r1 != r7 goto 9f; \ - r7 = 0x11100000111 ll; \ - r1 = 0x11000000111 ll; \ - .8byte %[fetch_and_dw]; /* r1 = atomic_fetch_and((u64 *)(r6 + 8), r1) */ \ - r0 = 2; \ - if r1 != r7 goto 9f; \ - r7 = 0x11000000111 ll; \ - r1 = 0x101 ll; \ - .8byte %[fetch_xor_dw]; /* r1 = atomic_fetch_xor((u64 *)(r6 + 8), r1) */ \ - r0 = 3; \ - if r1 != r7 goto 9f; \ - r7 = 0x11000000010 ll; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != r7 goto 9f; \ - *(u32 *)(r6 + 16) = 0x110; \ - r1 = 0xdead000000000011 ll; \ - .8byte %[fetch_or_w]; /* r1 = atomic_fetch_or((u32 *)(r6 + 16), r1) */ \ - r0 = 5; \ - if r1 != 0x110 goto 9f; \ - r1 = 0xdead000000000101 ll; \ - .8byte %[fetch_and_w]; /* r1 = atomic_fetch_and((u32 *)(r6 + 16), r1) */ \ - r0 = 6; \ - if r1 != 0x111 goto 9f; \ - r1 = 0xdead000000000011 ll; \ - .8byte %[fetch_xor_w]; /* r1 = atomic_fetch_xor((u32 *)(r6 + 16), r1) */ \ - r0 = 7; \ - if r1 != 0x101 goto 9f; \ - r2 = *(u32 *)(r6 + 16); \ - r0 = 8; \ - if r2 != 0x110 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)), - __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)), - __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching and/or/xor with the operand in r0") -__success __retval(0) -__load_if_JITed() -__naked void fetch_and_or_xor_r0(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = 0x11000000110 ll; \ - *(u64 *)(r6 + 8) = r7; \ - r0 = 0x1100000011 ll; \ - .8byte %[fetch_or_dw]; /* r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != r7 goto 9f; \ - r7 = 0x11100000111 ll; \ - r0 = 0x11000000111 ll; \ - .8byte %[fetch_and_dw]; /* r0 = atomic_fetch_and((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 2; \ - if r2 != r7 goto 9f; \ - r7 = 0x11000000111 ll; \ - r0 = 0x101 ll; \ - .8byte %[fetch_xor_dw]; /* r0 = atomic_fetch_xor((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 3; \ - if r2 != r7 goto 9f; \ - r7 = 0x11000000010 ll; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != r7 goto 9f; \ - *(u32 *)(r6 + 16) = 0x110; \ - r0 = 0xdead000000000011 ll; \ - .8byte %[fetch_or_w]; /* r0 = atomic_fetch_or((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 5; \ - if r2 != 0x110 goto 9f; \ - r0 = 0xdead000000000101 ll; \ - .8byte %[fetch_and_w]; /* r0 = atomic_fetch_and((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 6; \ - if r2 != 0x111 goto 9f; \ - r0 = 0xdead000000000011 ll; \ - .8byte %[fetch_xor_w]; /* r0 = atomic_fetch_xor((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 7; \ - if r2 != 0x101 goto 9f; \ - r2 = *(u32 *)(r6 + 16); \ - r0 = 8; \ - if r2 != 0x110 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)), - __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)), - __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)) - : __clobber_all); -} - -/* - * The page is gone, so the access faults. The fetched register has to come back - * 0 and everything else has to be left alone, r0 included. Reading the address - * again has to give 0 as well: had the fault been recovered by the arena mapping - * its zeroed scratch page instead of by the JIT's exception table entry, the - * access would have succeeded and left a 1 there. - */ -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching or through a number to a freed page") -__success __retval(0) -__load_if_JITed() -__naked void fetch_or_freed_page(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x55; \ - r1 = %[arena] ll; \ - r2 = r6; \ - r3 = 1; \ - call %[bpf_arena_free_pages]; \ - r0 = r6; \ - r0 ^= 0x55; \ - r1 = 1; \ - .8byte %[fetch_or_dw]; /* faults: r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ - r2 = r6; \ - r2 ^= 0x55; \ - if r0 == r2 goto 1f; \ - r0 = 1; \ - goto 9f; \ -1: r0 = 2; \ - if r1 != 0 goto 9f; \ - r3 = *(u64 *)(r6 + 8); \ - r0 = 3; \ - if r3 != 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_arena_free_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching or with the operand in r0 to a freed page") -__success __retval(0) -__load_if_JITed() -__naked void fetch_or_r0_freed_page(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x55; \ - r1 = %[arena] ll; \ - r2 = r6; \ - r3 = 1; \ - call %[bpf_arena_free_pages]; \ - r0 = 1; \ - .8byte %[fetch_or_dw]; /* faults: r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != 0 goto 9f; \ - r3 = *(u64 *)(r6 + 8); \ - r0 = 2; \ - if r3 != 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_arena_free_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: cmpxchg through r0") -__success __retval(0) -__load_if_JITed() -__naked void cmpxchg_r0(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - /* The address is in r0. The value at the address is not equal to it. */ \ - *(u64 *)(r6 + 0) = 3; \ - r0 = r6; \ - r1 = 5; \ - .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r0 + 0, r0, r1) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != 3 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 2; \ - if r2 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_0, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: xchg into the register that holds the address") -__success __retval(0) -__load_if_JITed() -__naked void xchg_into_base(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 0) = 3; \ - r1 = r6; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ - r0 = 1; \ - if r1 != 3 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 2; \ - if r2 != r6 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: xchg into the register that holds a pointer to stack") -__failure __msg("misaligned access off (0x0; 0xffffffffffffffff)+0 size 8") -__naked void xchg_into_stack_ptr(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u64 *)(r10 - 8) = r1; \ - r1 = r10; \ - r1 += -8; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ - r0 = 0; \ - exit; \ -" : - : __imm_addr(arena), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) - : __clobber_all); -} - -#ifdef CAN_USE_LOAD_ACQ_STORE_REL - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load-acquire and store-release") -__success __retval(0) -__load_if_JITed() -__naked void load_acq_store_rel(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - r1 = 0x1234; \ - .8byte %[store_release_insn]; /* store_release((u64 *)(r6 + 8), r1) */ \ - .8byte %[load_acquire_insn]; /* r2 = load_acquire((u64 *)(r6 + 8)) */ \ - r0 = 1; \ - if r2 != 0x1234 goto 9f; \ - .8byte %[load_acquire8_insn]; /* w2 = load_acquire((u8 *)(r6 + 8)) */ \ - r0 = 2; \ - if r2 != 0x34 goto 9f; \ - r0 = 3; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(store_release_insn, - BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)), - __imm_insn(load_acquire8_insn, - BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)) - : __clobber_all); -} - -#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number that is not an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void not_in_arena(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - /* nothing is allocated: all loads read 0, stores are dropped */ \ - r6 = 0xdeadbeef00000000 ll; \ - r0 = 1; \ - r2 = *(u64 *)(r6 + 0); \ - if r2 != 0 goto 9f; \ - r0 = 2; \ - r2 = *(u8 *)(r6 - 32768); \ - if r2 != 0 goto 9f; \ - r6 = 0x12345678ffffffff ll; \ - r0 = 3; \ - r2 = *(u64 *)(r6 + 32760); \ - if r2 != 0 goto 9f; \ - *(u64 *)(r6 + 32760) = 1; \ - *(u8 *)(r6 + 32767) = r2; \ - r1 = 1; \ - lock *(u64 *)(r6 + 32760) += r1; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: store through the address of the stack as a number") -__success __retval(0) -__load_if_JITed() -__naked void stack_addr_as_number(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - *(u64 *)(r10 - 8) = 5; \ - r6 = r10; \ - r6 |= 0; \ - /* r6 is a number now. The store goes to arena, not to the stack. */ \ - *(u64 *)(r6 - 8) = 7; \ - r1 = 9; \ - *(u64 *)(r6 - 8) = r1; \ - lock *(u64 *)(r6 - 8) += r1; \ - r2 = *(u64 *)(r10 - 8); \ - r0 = 1; \ - if r2 != 5 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: 64-bit math on the address stays 64-bit") -__success __retval(0) -__load_if_JITed() -__naked void alu64_after_access(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r7 = r6; \ - r7 += 8; \ - r7 -= r6; \ - r0 = 1; \ - if r7 != 8 goto 9f; \ - r7 = r6; \ - r7 >>= 32; \ - r0 = 2; \ - if r7 == 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: st of an immediate changes no register") -__success __retval(0) -__load_if_JITed() -__naked void st_keeps_regs(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r0 = r6; \ - r1 = 0x1111; \ - r2 = 0x2222; \ - *(u64 *)(r0 + 0) = 5; \ - r3 = r0; \ - r0 = 1; \ - if r3 != r6 goto 9f; \ - r0 = 2; \ - if r1 != 0x1111 goto 9f; \ - r0 = 3; \ - if r2 != 0x2222 goto 9f; \ - r0 = 0x3333; \ - r1 = r6; \ - *(u32 *)(r1 + 8) = -7; \ - r3 = r0; \ - r0 = 4; \ - if r3 != 0x3333 goto 9f; \ - r0 = 5; \ - if r1 != r6 goto 9f; \ - r0 = 6; \ - r3 = *(u64 *)(r6 + 0); \ - if r3 != 5 goto 9f; \ - r0 = 7; \ - r3 = *(u64 *)(r6 + 8); \ - r4 = 0xfffffff9 ll; \ - if r3 != r4 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: access through a number with garbage in the upper half") -__success __retval(0) -__load_if_JITed() -__naked void st_value_garbage(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0xdeadbeef00001000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - *(u64 *)(r8 + 0) = 1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - *(u8 *)(r8 + 1) = 1; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - *(u32 *)(r8 + 4) = r1; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r2 = *(u16 *)(r8 + 2); \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 5; \ - if r1 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: access through a number without the upper half") -__success __retval(0) -__load_if_JITed() -__naked void st_value_small(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0x1000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - *(u64 *)(r8 + 0) = 1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - *(u8 *)(r8 + 1) = 1; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - *(u32 *)(r8 + 4) = r1; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r2 = *(u16 *)(r8 + 2); \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 5; \ - if r1 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: atomics through a number that is not an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void atomic_value(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0xdeadbeef00001000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - lock *(u64 *)(r8 + 0) += r1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - .8byte %[fetch_add_insn]; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - .8byte %[xchg_insn]; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r0 = 0; \ - .8byte %[cmpxchg_insn]; \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, - BPF_REG_8, BPF_REG_1, 0)), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_8, BPF_REG_1, 0)), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_8, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number and pointer to arena at the same insn") -__success __retval(0) -__load_if_JITed() -__naked void mixed_number_arena(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x1234; \ - /* a number that the verifier does not know, 0 at run time */ \ - r7 = *(u64 *)(r6 + 16); \ - r8 = r6; \ - if r7 != 0 goto 1f; \ - .8byte %[cast_kern_insn]; \ -1: *(u8 *)(r8 + 0) = 1; \ - *(u8 *)(r8 + 1) = r7; \ - r2 = *(u8 *)(r8 + 1); \ - lock *(u64 *)(r8 + 24) += r7; \ - r0 = 0; \ - if r7 != 0 goto 9f; \ - /* pointer to arena only: JIT adds all 64 bits of r8 to the base */ \ - r0 = 2; \ - r2 = *(u64 *)(r8 + 8); \ - if r2 != 0x1234 goto 9f; \ - r0 = 3; \ - r2 = *(u8 *)(r6 + 0); \ - if r2 != 1 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(cast_kern_insn, BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, - BPF_REG_8, BPF_REG_8, 1, 1)) - : __clobber_all); -} - -SEC("syscall") -__description("arena_scalar: no flag, no access through a number") -__failure __msg("R6 invalid mem access 'scalar'") -__naked void no_flag(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r6 = 0x100000000000 ll; \ - r0 = *(u64 *)(r6 + 0); \ - exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: pointer that is NULL is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void null_ptr(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u32 *)(r10 - 4) = r1; \ - r2 = r10; \ - r2 += -4; \ - r1 = %[hash] ll; \ - call %[bpf_map_lookup_elem]; \ - r1 = r0; \ - r0 = 1; \ - if r1 != 0 goto 9f; \ - /* nothing is allocated: the load reads 0 */ \ - r0 = *(u64 *)(r1 + 0); \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: pointer that is NULL with an offset is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void null_ptr_off(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u32 *)(r10 - 4) = r1; \ - r2 = r10; \ - r2 += -4; \ - r1 = %[hash] ll; \ - call %[bpf_map_lookup_elem]; \ - r1 = r0; \ - r0 = 1; \ - if r1 != 0 goto 9f; \ - r1 += 8; \ - *(u64 *)(r1 + 0) = 5; \ - r0 = *(u64 *)(r1 + 0); \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number that is less than a page is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void small_number(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - call %[bpf_get_prandom_u32]; \ - r1 = r0; \ - r1 &= 0xfff; \ - r0 = *(u64 *)(r1 + 0); \ - exit; \ -" : - : __imm_addr(arena), - __imm(bpf_get_prandom_u32) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number is not a pointer for a helper") -__failure __msg("R2 type=scalar expected=") -__naked void helper_arg(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = %[hash] ll; \ - r2 = r6; \ - call %[bpf_map_lookup_elem]; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_arena_alloc_pages), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number and pointer to stack at the same insn") -__failure __msg("same insn cannot be used with different pointers") -__load_if_JITed() -__naked void mixed_number_stack(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r10 - 8) = 0; \ - call %[bpf_get_prandom_u32]; \ - if w0 != 0 goto 1f; \ - r6 = r10; \ - r6 += -8; \ -1: r0 = *(u64 *)(r6 + 0); \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_get_prandom_u32) - : __clobber_all); -} - -static int st_cb(__u64 idx, void *ctx) -{ - volatile long *p = *(volatile long **)ctx; - - p[idx] = 7; - return 0; -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: store through a number in a callback") -__success __retval(0) -__load_if_JITed() -int st_in_callback(void *unused) -{ - volatile long *p = bpf_arena_alloc_pages(&arena, NULL, 1, -1, 0); - - if (!p) - return 100; - bpf_loop(4, st_cb, &p, 0); - return p[0] + p[3] + p[4] - 14; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 824e5912ebcb..d24e4fedae46 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -590,8 +590,6 @@ static int parse_test_spec(struct test_loader *tester, update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); - } else if (strcmp(val, "BPF_F_ARENA_SCALAR") == 0) { - update_flags(&spec->prog_flags, BPF_F_ARENA_SCALAR, clear); } else /* assume numeric value */ { err = parse_int(val, &flags, "test prog flags"); if (err) From b62dee3a8bb5d6c53eef8c727aceae369fbdb3a8 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:00 +0000 Subject: [PATCH 32/48] bpf: Remove BPF_F_ARENA_SCALAR Rust has no address spaces. Thus a prog in Rust accesses arena through scalars. With BPF_F_ARENA_SCALAR the verifier accepts each load and each store through a scalar as an access to arena. The verifier rewrites one insn at a time. Thus it must add one insn to each access. The compiler knows the source of each pointer. It can emit one addr_space_cast for each pointer, as it does for the __arena pointers of C. flag: one move for each access cast: one cast for each pointer ax = (u32)r1 r1 = addr_space_cast(r1, 0, 1) r2 = *(u64 *)(ax + 0) r2 = *(u64 *)(r1 + 0) ax = (u32)r1 r3 = *(u64 *)(r1 + 8) r3 = *(u64 *)(ax + 8) 20 progs of rust-bpf, with the flag and with the casts that a prototype of an LLVM pass emits: flag casts moves or casts 1568 571 insns 14864 13938 jited bytes 72351 69111 No released kernel has the flag. LLVM does not emit the casts for Rust at this time. Until it does, Rust progs that access arena do not load. scx_mavd and scx_nitosis set the flag too. They are experimental schedulers in C in sched-ext/scx, and they use the flag to omit __arena. They must use __arena again. This reverts commit 4c651a91bdfc ("bpf: Treat load and store through a number as arena access"). One part stays: BPF_REG_AX in reg2pt_regs[] of the x86 JIT. The fetching AND/OR/XOR atomics in arena use it now. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-6-alexei.starovoitov@gmail.com --- arch/arm64/net/bpf_jit_comp.c | 5 --- arch/x86/net/bpf_jit_comp.c | 5 --- include/linux/bpf_verifier.h | 2 -- include/linux/filter.h | 1 - include/uapi/linux/bpf.h | 6 ---- kernel/bpf/core.c | 9 +---- kernel/bpf/fixups.c | 39 --------------------- kernel/bpf/syscall.c | 4 --- kernel/bpf/verifier.c | 64 ++++++++-------------------------- tools/include/uapi/linux/bpf.h | 6 ---- 10 files changed, 16 insertions(+), 125 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index aaa34c739004..33632fb20997 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -3502,11 +3502,6 @@ bool bpf_jit_supports_arena(void) return true; } -bool bpf_jit_supports_arena_scalar(void) -{ - return true; -} - bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index fd65ac84cfc7..7bfb7210484e 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -5240,11 +5240,6 @@ bool bpf_jit_supports_arena(void) return true; } -bool bpf_jit_supports_arena_scalar(void) -{ - return true; -} - bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return true; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index e51984cb3035..e3f7a0b3bad3 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -782,7 +782,6 @@ struct bpf_insn_aux_data { u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */ u64 is_iter_next:1; /* bpf_iter__next() kfunc call */ u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */ - u64 arena_scalar:1; /* ldx/stx/st/atomic through a number, it's an address in arena */ u64 bb_end:1; /* last instruction of a basic block */ u64 need_scev:1; u64 alu_state:5; /* used in combination with alu_limit */ @@ -1074,7 +1073,6 @@ struct bpf_verifier_env { bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ bool test_reg_invariants; /* fail verification on register invariants violations */ - bool arena_scalar; /* load and store through a number is arena access */ struct bpf_verifier_state *cur_state; /* current verifier state */ /* Search pruning optimization, array of list_heads for * lists of struct bpf_verifier_state_list. diff --git a/include/linux/filter.h b/include/linux/filter.h index e0218e677a1d..98254742ae37 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1287,7 +1287,6 @@ bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); -bool bpf_jit_supports_arena_scalar(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); bool bpf_jit_supports_large_stack(void); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index f29b0aad440c..8d7228834305 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1357,12 +1357,6 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) -/* - * Load and store through a number is an access to the arena of the program - * at the low 32 bits of the number. It's for programs written in Rust. - */ -#define BPF_F_ARENA_SCALAR (1U << 8) - /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 0a8ba603f44b..7cf37690d41f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -2639,8 +2639,7 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) /* * Whether a function has an insn that works with the arena base, which a JIT * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on - * the final insns of the function: an access through an arena pointer, or - * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been + * the final insns of the function: an access through an arena pointer has been * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, * which the JIT rebases, has a function model that says so. A load of an @@ -3518,12 +3517,6 @@ bool __weak bpf_jit_supports_arena(void) return false; } -/* Whether JIT takes BPF_REG_AX as the address of arena access */ -bool __weak bpf_jit_supports_arena_scalar(void) -{ - return false; -} - bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return false; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index f062ef7dd943..41bc37e205e0 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -44,31 +44,6 @@ static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *i return false; } -/* - * The insn accesses arena through a number. JITs add the base of arena - * to the register as it is, so the access goes through the low 32 bits of - * the number in BPF_REG_AX. Registers of the program are not changed: the - * number may be compared or stored later, and on another path the register - * may be PTR_TO_ARENA. - * - * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate - * of st through a number is not blinded. - */ -static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) -{ - bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); - struct bpf_insn *patch = buf; - - *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); - *patch = *insn; - if (load) - patch->src_reg = BPF_REG_AX; - else - patch->dst_reg = BPF_REG_AX; - patch++; - return patch - buf; -} - /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -1880,20 +1855,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); - if (env->insn_aux_data[i + delta].arena_scalar) { - cnt = arena_scalar_access(insn, insn_buf); - - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - prog = new_prog; - env->prog = prog; - insn = prog->insnsi + i + delta; - goto next_insn; - } - /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 90bc7230afa8..60c4d84b9aba 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2971,13 +2971,9 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_TEST_REG_INVARIANTS | - BPF_F_ARENA_SCALAR | BPF_F_TOKEN_FD)) return -EINVAL; - if ((attr->prog_flags & BPF_F_ARENA_SCALAR) && !bpf_jit_supports_arena_scalar()) - return -EOPNOTSUPP; - bpf_prog_load_fixup_attach_type(attr); if (attr->prog_flags & BPF_F_TOKEN_FD) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a0a5f075d201..c1070aba621b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5500,19 +5500,6 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } -/* - * There are no address spaces in Rust, addresses of arena are plain numbers. - * When the program is loaded with BPF_F_ARENA_SCALAR and has an arena a load or - * a store through a number is an access to arena at the low 32 bits of it, - * like the access through PTR_TO_ARENA is. The register stays a number. - */ -static bool is_arena_scalar(struct bpf_verifier_env *env, int regno) -{ - const struct bpf_reg_state *reg = reg_state(env, regno); - - return reg->type == SCALAR_VALUE && env->arena_scalar && env->prog->aux->arena; -} - static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn) { @@ -5543,7 +5530,7 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_sk_reg(env, regno)) return false; - if (is_arena_reg(env, regno) || is_arena_scalar(env, regno)) + if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); if (is_load_acq_unsafe(env, regno, insn)) return false; @@ -7524,22 +7511,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch); -/* - * Returns the register to check the access of the current insn with. - * For a number in a program with an arena that is 'arena'. - */ -static struct bpf_reg_state *mem_access_reg(struct bpf_verifier_env *env, int regno, - struct bpf_reg_state *arena) -{ - if (!is_arena_scalar(env, regno)) - return cur_regs(env) + regno; - - memset(arena, 0, sizeof(*arena)); - arena->type = PTR_TO_ARENA; - env->insn_aux_data[env->insn_idx].arena_scalar = true; - return arena; -} - static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) @@ -7547,7 +7518,6 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state arena, *src_reg; enum bpf_reg_type src_reg_type; int err; @@ -7569,16 +7539,16 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - src_reg = mem_access_reg(env, insn->src_reg, &arena); - src_reg_type = src_reg->type; + src_reg_type = regs[insn->src_reg].type; /* * check_stack_read_fixed_off() may refine the modification's origin to * the source stack slot. */ bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); - err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), - insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, + argno_from_reg(insn->src_reg), insn->off, + BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); /* * Paths through a kernel struct pointer and through other pointers @@ -7606,7 +7576,6 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; int err; @@ -7628,12 +7597,12 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - dst_reg_type = dst_reg->type; + dst_reg_type = regs[insn->dst_reg].type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), - insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, + err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, + argno_from_reg(insn->dst_reg), insn->off, + BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -7643,7 +7612,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { - struct bpf_reg_state arena, *dst_reg; + struct bpf_reg_state *dst_reg; int load_reg; int err; @@ -7688,9 +7657,6 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - /* load_reg may be dst_reg. Look at dst_reg before it's marked as unknown. */ - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - load_reg = bpf_atomic_load_reg(insn); if (load_reg >= 0) { /* check and record load of old value */ @@ -7699,6 +7665,8 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } + dst_reg = cur_regs(env) + insn->dst_reg; + /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ @@ -20289,17 +20257,16 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_stack_arg_write(env, state, insn->off, NULL); } - struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - dst_reg_type = dst_reg->type; + dst_reg_type = cur_regs(env)[insn->dst_reg].type; - err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), + err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, + argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -23935,7 +23902,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; - env->arena_scalar = attr->prog_flags & BPF_F_ARENA_SCALAR; env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index f29b0aad440c..8d7228834305 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1357,12 +1357,6 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) -/* - * Load and store through a number is an access to the arena of the program - * at the low 32 bits of the number. It's for programs written in Rust. - */ -#define BPF_F_ARENA_SCALAR (1U << 8) - /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ From 701edc66772b41474f691664efc179d0de840ed1 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:01 +0000 Subject: [PATCH 33/48] bpf, x86: Remove r0_on_stack from fetching atomics in arena x86 has no fetching AND/OR/XOR. Thus the JIT emits a loop with cmpxchg. cmpxchg uses RAX, and RAX holds BPF R0. Thus the JIT saves R0 before the loop. prog the address is in the JIT saves R0 in -------------------------- ------------------ ------------------- without the flag a BPF register BPF_REG_AX with BPF_F_ARENA_SCALAR BPF_REG_AX the stack The flag is removed. Remove the second variant. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-7-alexei.starovoitov@gmail.com --- arch/x86/net/bpf_jit_comp.c | 58 +++++++++---------------------------- 1 file changed, 14 insertions(+), 44 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 7bfb7210484e..d90b6d238b3a 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1745,26 +1745,20 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, /* * A fetching AND/OR/XOR can't be implemented with a single x86 insn, so do a * CMPXCHG loop. @index_reg is X86_REG_R12 for an arena access or -1 otherwise. - * Unless @r0_on_stack, the caller has substituted BPF_REG_AX for R0 in - * @dst_reg and @src_reg. + * The caller has saved R0 in BPF_REG_AX and has substituted BPF_REG_AX for R0 + * in @dst_reg and @src_reg. * * Both the load and the CMPXCHG can fault when accessing the arena, so their * addresses are handed back in @fault[0] and @fault[1]. A fault has to resume * at @resume, which is past the loop and past the move that delivers the old * value, but before R0 is restored. Both are NULL for the non-arena case, * which needs no fixup. - * - * R0 is expected to be saved in BPF_REG_AX by the caller, or pushed on the - * stack if @r0_on_stack. In that case an R0 operand stays in the stack slot, - * RAX ends up holding the old value that R0 is to return, and the slot is - * dropped rather than popped. */ static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, - u32 src_reg, int index_reg, bool r0_on_stack, + u32 src_reg, int index_reg, u8 **fault, u8 **resume) { bool is64 = BPF_SIZE(insn->code) == BPF_DW; - bool operand_on_stack = r0_on_stack && src_reg == BPF_REG_0; u8 *branch_target, *prog = *pprog; int err; @@ -1783,15 +1777,7 @@ static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, * Perform the (commutative) operation locally, put the result in * the AUX_REG. */ - if (operand_on_stack) { - /* mov r11, [rsp]: the operand is the R0 saved there, RAX is the old value */ - if (is64) - EMIT4(0x4C, 0x8B, 0x1C, 0x24); - else - EMIT4(0x44, 0x8B, 0x1C, 0x24); - } else { - emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); - } + emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); maybe_emit_mod(&prog, AUX_REG, src_reg, is64); EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], add_2reg(0xC0, AUX_REG, src_reg)); @@ -1810,20 +1796,14 @@ static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, /* ZF tells us whether we won the race. If it's cleared we need to try again. */ EMIT2(X86_JNE, -(prog - branch_target) - 2); - /* Return the pre-modification value, which RAX already is for R0 */ - if (!operand_on_stack) - emit_mov_reg(&prog, is64, src_reg, BPF_REG_0); + /* Return the pre-modification value */ + emit_mov_reg(&prog, is64, src_reg, BPF_REG_0); if (resume) *resume = prog; /* Restore R0 after clobbering RAX */ - if (!r0_on_stack) - emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); - else if (operand_on_stack) - EMIT2(0x41, 0x5B); /* pop r11, dropping the operand */ - else - EMIT1(0x58); /* pop rax */ + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); *pprog = prog; return 0; @@ -3354,7 +3334,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * } if (is_atomic_fetch) { err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, - real_src_reg, -1, false, + real_src_reg, -1, NULL, NULL); if (err) return err; @@ -3383,29 +3363,19 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: if (is_atomic_fetch_op(insn)) { u32 real_src_reg = src_reg, real_dst_reg = dst_reg; - /* - * An access through a number has its address in - * BPF_REG_AX (see arena_scalar_access()), so R0 is - * saved on the stack instead, as div and mod do. - */ - bool r0_on_stack = dst_reg == BPF_REG_AX; u8 *fault[2], *resume; int j; /* Will need RAX as a CMPXCHG operand so save R0 */ - if (r0_on_stack) { - EMIT1(0x50); /* push rax */ - } else { - emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); - if (src_reg == BPF_REG_0) - real_src_reg = BPF_REG_AX; - if (dst_reg == BPF_REG_0) - real_dst_reg = BPF_REG_AX; - } + emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); + if (src_reg == BPF_REG_0) + real_src_reg = BPF_REG_AX; + if (dst_reg == BPF_REG_0) + real_dst_reg = BPF_REG_AX; err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, real_src_reg, X86_REG_R12, - r0_on_stack, fault, &resume); + fault, &resume); if (err) return err; From 3f9924747c4422176346c34b5dff44ba8bca5180 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:02 +0000 Subject: [PATCH 34/48] bpf: Clear delta of a register that scalar evolution widens After 'r3 = r8; r3 += 4' the verifier knows that r3 == r8 + 4. The two registers have the same id, and r3 keeps the difference in reg->delta. When the prog compares one of them with a constant, sync_linked_regs() calculates the range of the other register from the difference of the two deltas. At a loop head scalar evolution widens a register: it replaces the register with its range over all iterations. Then the register is not linked to other registers. scratch_widened_reg_id() clears reg->id, but it does not clear reg->delta. A subsequent copy of the register gets the stale delta. Then the range of the copy is incorrect by this delta. r3 = r8 r3 += 4 ; R3=scalar(id=1+4) 1: ; r3 is widened: no id, delta is still 4 r4 = r3 ; id=2 and delta 4 in both r4 += 1 ; R4=scalar(id=2+1) if r3 > 10 goto 3f ; R3 <= 10, R4 = R3 + 1 - 4 <= 7 if r4 <= 7 goto 2f ... ; not verified, r4 is up to 11 here The verifier does not verify code that can run. Use clear_scalar_id(). It clears id and delta together. Move it to bpf_verifier.h, so that scev.c can call it. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-8-alexei.starovoitov@gmail.com --- include/linux/bpf_verifier.h | 6 ++++++ kernel/bpf/scev.c | 2 +- kernel/bpf/verifier.c | 6 ------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index e3f7a0b3bad3..7baeb831281f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1313,6 +1313,12 @@ static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int ins return env->insn_aux_data[insn_idx].force_checkpoint; } +static inline void clear_scalar_id(struct bpf_reg_state *reg) +{ + reg->id = 0; + reg->delta = 0; +} + static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx) { env->insn_aux_data[idx].calls_callback = true; diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 04d4962448e2..3f7534cda47a 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2391,7 +2391,7 @@ static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_ { switch (base_type(reg->type)) { case SCALAR_VALUE: - reg->id = 0; + clear_scalar_id(reg); break; case PTR_TO_PACKET: case PTR_TO_PACKET_META: diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index c1070aba621b..8139e41ace00 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3824,12 +3824,6 @@ static bool __is_pointer_value(bool allow_ptr_leaks, return is_pointer_regtype(reg->type); } -static void clear_scalar_id(struct bpf_reg_state *reg) -{ - reg->id = 0; - reg->delta = 0; -} - static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, struct bpf_reg_state *src_reg) { From 165115760433adf29c76190db5ecfe781d6c9221 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:03 +0000 Subject: [PATCH 35/48] selftests/bpf: Test that a widened register is not linked Scalar evolution widens a register that has delta 4. Then the prog copies the register. With the stale delta the verifier finds that a read of uninitialized r9 is dead code, and it loads the prog. See "bpf: Clear delta of a register that scalar evolution widens". Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-9-alexei.starovoitov@gmail.com --- .../selftests/bpf/progs/verifier_scev.c | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 37434efd1490..ddc9667bb078 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -1377,6 +1377,40 @@ __naked __noinline __used static int probe_read_stack(void) : __clobber_all); } +/* + * r3 = r8 + 4 is linked to r8 when it enters the loop. Widening drops the link. If + * it leaves the 4 behind then r4 = r3 + 1 is taken for r3 - 3 when r3 is compared: + * r4 <= 7 instead of r4 <= 11, and the read of r9 looks like dead code. + */ +SEC("socket") +__failure __msg("R9 !read_ok") +__naked void widened_reg_is_not_linked(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r8 = r0;" + "r8 &= 0xff;" + "r3 = r8;" + "r3 += 4;" + "r1 = 0;" +"1:" + "r4 = r3;" + "r4 += 1;" + "if r3 > 10 goto 3f;" + "if r4 <= 7 goto 2f;" + "r0 = *(u64 *)(r9 + 0);" +"2:" + "r3 += 1;" + "r1 += 1;" + "if r1 < 2 goto 1b;" +"3:" + "r0 = r8;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + /* The unknown SCEV of an unbound STACK_MISC slot must not prevent widening. */ SEC("tracepoint") __success From 2ad395a3847113289e443aafd51f2a751bc32ea2 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:04 +0000 Subject: [PATCH 36/48] selftests/bpf: Wait for the last maps of map_kptr to be freed map_kptr_race/htab_leak fails occasionally in a complete run of test_progs: test_htab_leak:FAIL:htab refcount unexpected htab refcount: actual 6 != expected 2 This is a race between two tests. It is not a leak in the kernel. - map_kptr runs immediately before htab_leak. The two tests examine the refcount of the same test object. - The kernel frees maps asynchronously. Thus the maps drop their references some time after close(). - map_kptr waits for that after each subtest, but not after the last one. Wait after the last subtest too. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-10-alexei.starovoitov@gmail.com --- tools/testing/selftests/bpf/prog_tests/map_kptr.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/map_kptr.c b/tools/testing/selftests/bpf/prog_tests/map_kptr.c index 17e707dddda8..f103f8cd458b 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/map_kptr.c @@ -225,6 +225,11 @@ void serial_test_map_kptr(void) /* Observe refcount dropping to 1 on map release. */ test_map_kptr_success(true); + + /* The next test counts the same references */ + ASSERT_OK(kern_sync_rcu_tasks_trace(skel), "sync rcu_tasks_trace"); + ASSERT_OK(kern_sync_rcu(), "sync rcu"); + wait_for_map_release(); } rcu_tasks_trace_gp__destroy(skel); From 3c0ca9f9a98e6b8bbaea13c5e39613237d50fe47 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:05 +0000 Subject: [PATCH 37/48] selftests/bpf: Avoid a page fault in rcu_reentry while the handler is armed The test arms a handler at rcu_segcblist_enqueue() for its thread. Then it deletes an element of a task storage map. The delete gets to rcu_segcblist_enqueue(), and the handler does a nested delete there. The handler fires at the first rcu_segcblist_enqueue() in the thread. A page fault can be the first: skel->bss->target_pid = tid the handler is armed the next insn is in a page that is not in memory asm_exc_page_fault filemap_fault v9fs_issue_read p9_req_put irq_work_run rcu_defer_drain rcu_segcblist_enqueue the handler deletes the element bpf_map_delete_elem() -ENOENT test_rcu_reentry:FAIL:boot_delete unexpected error: -2 (errno 2) The result depends on the layout of test_progs. In the binary that failed, test_rcu_reentry() crosses the end of a page immediately after the store. This binary fails each time when it is on 9p. A copy of the same file on ext4 passes. Move the store and the delete to a function. Call it one time without a pid and with a bad fd. This call brings the text and .bss into memory, and it does not arm the handler. Fixes: 0464f587c61a ("selftests/bpf: Add a call_srcu() re-entry reproducer") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-11-alexei.starovoitov@gmail.com --- .../selftests/bpf/prog_tests/rcu_reentry.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c b/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c index de23a14b3d40..10b4ea913921 100644 --- a/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c +++ b/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c @@ -23,10 +23,17 @@ static bool have_tree_srcu(void) return kallsyms_find("srcu_expedite_current", &addr) == 0; } +/* Arm the handler for this thread, then trigger call_rcu_tasks_trace(). */ +static noinline int arm_and_delete(struct rcu_reentry *skel, int pid, int map_fd, int *pidfd) +{ + skel->bss->target_pid = pid; + return bpf_map_delete_elem(map_fd, pidfd); +} + void test_rcu_reentry(void) { struct rcu_reentry *skel; - int err, pidfd = -1, map_fd; + int err, pidfd = -1, bad_fd = -1, map_fd; cpu_set_t set, old_set; bool affinity_saved; __u64 val = 1; @@ -63,9 +70,13 @@ void test_rcu_reentry(void) if (!ASSERT_OK(err, "boot_create")) goto restore; - /* Arm the handler for this thread, then trigger call_rcu_tasks_trace(). */ - skel->bss->target_pid = syscall(__NR_gettid); - err = bpf_map_delete_elem(map_fd, &pidfd); + /* + * The handler fires at the first rcu_segcblist_enqueue() in this thread, and it deletes + * the element. A page fault can get there too, when the file system reads the page. + * Go through the same code first, without a pid, to bring its text and .bss in. + */ + arm_and_delete(skel, 0, map_fd, &bad_fd); + err = arm_and_delete(skel, syscall(__NR_gettid), map_fd, &pidfd); if (!ASSERT_OK(err, "boot_delete")) goto restore; From 41750e5d682ab0703a9d2af3ec31a508b253b38f Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:06 +0000 Subject: [PATCH 38/48] bpf: Don't widen a pointer to a buffer in scalar evolution Scalar evolution widens counted loops. Since that change the verifier rejects a loop over the per-cpu values of a map element: SEC("iter/bpf_map_elem") ... for (i = 0; i < n; i++) { out[i] = *(__u32 *)pptr; pptr += 8; } loop header at 12, widening r2 to 0..56 step 8 R2 invalid variable buffer offset: off=0, var_off=(0x0; 0x38) pptr is a pointer to a buffer (PTR_TO_BUF). Scalar evolution replaces its offset with the range over all iterations. But __check_buffer_access() accepts only a constant offset. Thus a prog cannot use a widened PTR_TO_BUF. Do not widen it. Then the verifier verifies the loop one iteration at a time, as it did before. loop before SCEV with SCEV this patch -------------------------------- ----------- ---------- ---------- counts in i, reads through pptr loads rejected loads counts in p ('p < end'), no read 1M insns loads 1M insns counts in p, reads through p 1M insns rejected 1M insns The verifier cannot decide a comparison of two pointers to a buffer. Thus it follows 'p < end' until the limit of 1M insns. bpf_iter_bpf_percpu_array_map.c and bpf_iter_bpf_percpu_hash_map.c have the first loop. They load today because num_cpus is volatile. The prog reads the bound again in each iteration. Scalar evolution cannot count such a loop, and it widens nothing. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-12-alexei.starovoitov@gmail.com --- kernel/bpf/scev.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 3f7534cda47a..6231d16aa311 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2379,7 +2379,6 @@ static bool is_widenable_reg_type(const struct bpf_reg_state *reg) case PTR_TO_PACKET: case PTR_TO_PACKET_META: case PTR_TO_MEM: - case PTR_TO_BUF: case PTR_TO_BTF_ID: return true; default: From 1b4ed112beb82a6adcd48a46fa8910c7184070e3 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:07 +0000 Subject: [PATCH 39/48] selftests/bpf: Test a loop that moves a pointer to a buffer ptr_buf moves one pointer to a buffer until it gets to a second pointer. It never reads through the pointer. It loads only if scalar evolution widens the pointer, and that is no longer the case. Change it to the loop that real progs have, for example bpf_iter_bpf_percpu_array_map.c. It counts in a register and reads through the pointer in each iteration. Without "bpf: Don't widen a pointer to a buffer in scalar evolution" the verifier rejects it with "invalid variable buffer offset". Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-13-alexei.starovoitov@gmail.com --- tools/testing/selftests/bpf/progs/verifier_scev.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index ddc9667bb078..cf7916e1fc48 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3517,21 +3517,23 @@ __naked void ptr_mem_other_reservation(void) : __clobber_all); } +/* An access to a buffer at a variable offset is rejected: the pointer must not be widened. */ SEC("iter/bpf_map_elem") __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("header_count is 4 ") -__msg("widening r6 to 0..3 step 1") +__msg("can't widen r6, expr is (+ r6 8)") __naked void ptr_buf(void) { asm volatile (" \ r6 = *(u64 *)(r1 + %[value]); \ if r6 == 0 goto 2f; \ - r7 = r6; \ - r7 += 4; \ -1: r6 += 1; \ - if r6 < r7 goto 1b; \ + r7 = 0; \ +1: r0 = *(u32 *)(r6 + 0); \ + r6 += 8; \ + r7 += 1; \ + if r7 < 4 goto 1b; \ 2: r0 = 0; \ exit; \ " : From 1536e9b4375f235596ab5b7d6aa0fc4573965c4e Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:08 +0000 Subject: [PATCH 40/48] bpf: Don't clamp a register whose type changed in the loop When a state returns to the head of a counted loop, bpf_clamp_scev_regs() replaces the range of each register that changes by a constant step. The new range is the range over all iterations. The function calculates it from the register in the state that entered the loop. For a pointer this range is the range of its offsets. A call in the loop can invalidate the pointer. bpf_skb_pull_data() does it to packet pointers, bpf_ringbuf_discard() does it to ringbuf memory. Then the register is an unknown scalar. Scalar evolution does not see that, and the clamp gives the offsets of the pointer to the scalar. r6 = bpf_ringbuf_reserve() ; R6=ringbuf_mem r7 = r6 1: if first iteration goto 2f r1 = map_value r1 += r6 ; accepted: R6 is 0 or 1 r0 = *(u8 *)(r1 + 0) goto 3f 2: bpf_ringbuf_discard(r7) ; R6=scalar() 3: r6 += 1 r9 += 1 if r9 < 2 goto 1b ; clamp: R6=scalar(umax=1) At run time r6 is the address of the ringbuf memory plus 1. The prog adds it to a pointer to a map value, and the kernel crashes in the access. Do not clamp a register that has a different type than at the entry of the loop. Such a state is not within the state at the head. Thus the verifier continues with it, and r6 stays an unknown scalar. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-14-alexei.starovoitov@gmail.com --- kernel/bpf/scev.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 6231d16aa311..89f96682af3f 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2897,6 +2897,14 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur is_unbound_scev_reg(env, entry_st, r)) continue; + /* + * A call in the loop can change the type: it invalidates packet pointers or + * releases a reference. The bounds are the offsets of the pointer at the entry. + * They say nothing about the value that is in the register now. + */ + if (reg->type != entry_reg->type) + continue; + linear_bounds(entry_reg, max_header_count, &coefs, &bounds); if (log->level & BPF_LOG_LEVEL2) { bpf_log(log, "loop header at %d, clamping ", insn_idx); From 41b06cfd1ceaf10e382b8f2d2d720e3e0abae10b Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:09 +0000 Subject: [PATCH 41/48] bpf: Reset the range of a packet pointer that scalar evolution clamps A compare with pkt_end sets 'range' in all packet pointers that have the same id. Then 'range - offset' bytes from the pointer are in the packet. When a state returns to the head of a counted loop, bpf_clamp_scev_regs() gives a new id to a packet pointer, and the offsets of all iterations. It does not change the range. But the range is valid only together with the old id and the old offsets. 1: ; R6=pkt(id=1,r=0,off=0..100) r5 = r6 r5 += 100 if r5 > pkt_end goto 2f ; R6=pkt(id=1,r=200,off=0..100) r6 += 100 ; R6=pkt(id=1,r=200,off=100..200) r9 += 1 if r9 < 2 goto 1b ; clamp: R6=pkt(id=2,r=200,off=0..100) after bytes from r6 that the verifier accepts ---------------------- --------------------------------------- the compare 100, correct r6 += 100 0, correct the clamp 100, incorrect. Nothing compared this r6 Usually the verifier prunes the state at the head, and the range has no effect. The verifier continues with the state when a different part of it is not within the state at the head. A flag in a 32-bit spill is sufficient. Then the prog reads and writes after pkt_end. A write to skb_shared_info crashes the kernel. Take the range that the pointer has at the entry of the loop. No compare in the loop found this range, so it is valid in all iterations. The state at the head has the same range, so the verifier prunes as before. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-15-alexei.starovoitov@gmail.com --- kernel/bpf/scev.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 89f96682af3f..7b82fd79f9cc 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -2914,6 +2914,13 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur bpf_log(log, "\n"); } scratch_widened_reg_id(env, reg); + /* + * The range of a packet pointer goes together with its id and with the offsets + * that it had in the compare with pkt_end. Both are replaced here. What holds for + * the new offsets is the range at the entry. + */ + if (type_is_pkt_pointer(reg->type)) + reg->range = max(entry_reg->range, 0); err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, bounds.base, bounds.step); if (err) From ed50365314826f1fcbed81e202ce475892ac5627 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:10 +0000 Subject: [PATCH 42/48] selftests/bpf: Test the clamp of pointers in scalar evolution test the loop result -------------------------------------- ------------------------- ------ clamp_released_pointer releases ringbuf memory reject clamp_invalidated_pkt_pointer calls bpf_skb_pull_data() reject clamp_pkt_pointer_keeps_no_range compares with pkt_end reject clamp_pkt_pointer_keeps_range_of_entry has no compare loads In the first three tests a flag in a 32-bit spill separates the first iteration from the second one. Scalar evolution does not widen the flag. Thus the verifier does not prune the state that returns to the head of the loop, and it verifies the second iteration with the clamped register. Without the fixes the three progs load. In the last test the compare with pkt_end is in front of the loop. Its range must stay valid in the loop. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-16-alexei.starovoitov@gmail.com --- .../selftests/bpf/progs/verifier_scev.c | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index cf7916e1fc48..fa01a616af9d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3773,4 +3773,167 @@ __naked void post_cond_jlt32_copy(void) : __clobber_all); } +/* + * r6 is a pointer to ringbuf memory with a step of 1. bpf_ringbuf_discard() in the first iteration + * makes a number of it. The offsets 0..1 of the pointer are not the range of this number: it's an + * address. The flag in the 32-bit spill is not widened, so the second iteration is verified + * on its own. + */ +SEC("socket") +__failure __msg("math between map_value pointer and register with unbounded min value") +__naked void clamp_released_pointer(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 12) = r1;" + "r2 = r10;" + "r2 += -12;" + "r1 = %[small_map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 9f;" + "r8 = r0;" + "r1 = %[ringbuf] ll;" + "r2 = 16;" + "r3 = 0;" + "call %[bpf_ringbuf_reserve];" + "if r0 == 0 goto 9f;" + "r6 = r0;" + "r7 = r0;" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + "r1 = r8;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" + "goto 3f;" +"2:" + "r1 = r7;" + "r2 = 0;" + "call %[bpf_ringbuf_discard];" +"3:" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 1;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), + __imm_addr(small_map), __imm_addr(ringbuf) + : __clobber_all); +} + +/* The same with a packet pointer and bpf_skb_pull_data() */ +SEC("tc") +__failure __msg("math between map_value pointer and register with unbounded min value") +__naked void clamp_invalidated_pkt_pointer(void) +{ + asm volatile ( + "r7 = r1;" + "r1 = 0;" + "*(u32 *)(r10 - 12) = r1;" + "r2 = r10;" + "r2 += -12;" + "r1 = %[small_map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 9f;" + "r8 = r0;" + "r6 = *(u32 *)(r7 + %[data]);" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + "r1 = r8;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" + "goto 3f;" +"2:" + "r1 = r7;" + "r2 = 0;" + "call %[bpf_skb_pull_data];" +"3:" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 1;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), __imm(bpf_skb_pull_data), __imm_addr(small_map), + __imm_const(data, offsetof(struct __sk_buff, data)) + : __clobber_all); +} + +/* + * The compare with data_end is for r6 of this iteration: 100 bytes from r6 are in the packet. Then + * r6 moves by 100. The range that the compare found is not for the r6 that returns to the head. + */ +SEC("tc") +__failure __msg("invalid access to packet") +__naked void clamp_pkt_pointer_keeps_no_range(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[data_end]);" + "r6 = *(u32 *)(r1 + %[data]);" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + /* second iteration: nothing compared this r6 with data_end */ + "r0 = *(u64 *)(r6 + 56);" +"2:" + "r5 = r6;" + "r5 += 100;" + "if r5 > r7 goto 9f;" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 100;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +/* A compare in front of the loop is for all iterations: the range at the entry stays */ +SEC("tc") +__success __retval(0) +__naked void clamp_pkt_pointer_keeps_range_of_entry(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[data_end]);" + "r6 = *(u32 *)(r1 + %[data]);" + "r5 = r6;" + "r5 += 40;" + "if r5 > r7 goto 9f;" + "r9 = 0;" +"1:" + "r0 = *(u64 *)(r6 + 0);" + "r6 += 8;" + "r9 += 1;" + "if r9 < 4 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; From 19caa38daf9def4a7c995ebf3bd0e89bce707652 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:11 +0000 Subject: [PATCH 43/48] bpf: Finish the insn in the state that maybe_fork_scalars() splits off When a register is -1 or 0, maybe_fork_scalars() splits the state in two before an 'and' or an 'or' with a constant. The result is more accurate for each value alone. The verifier continues with -1. It pushes a copy of the state with 0 and verifies the copy later, from the same insn. today this patch N: r1 &= 8 r1 is -1 or 0 N: r1 &= 8 r1 is -1 or 0 / \ / \ r1 = -1 r1 = 0 r1 = -1 r1 = 0 r1 &= 8 push to N r1 &= 8 r1 &= 8 | | push to N+1 N+1: ... N+1: ... At insn N the copy is a part of the state that it came from: 0 is in [-1, 0]. If pruning compares the two states, it drops the copy, and no one verifies the case r1 = 0. The first state has branches that are not explored. Usually pruning does not use such states. In a loop that scalar evolution counted it does, and the head of such a loop always keeps its state. Thus a fork at the head of a counted loop loses the case r1 = 0: r9 = a map value of 8 bytes r6 = -1 1: r6 &= 8 r6 is -1 or 0, only -1 is verified: 8 r6 ^= 8 0. It is 8 when r6 was 0. r1 = r9 r1 += r6 r0 = *(u8 *)(r1 + 0) r6 = -1 or 0 r7 += 1 if r7 < 100 goto 1b from 11 to 11: safe The verifier accepts this prog. The prog reads the byte that follows the map value. With a constant of 64 Tbyte in a register it reads an address that is not mapped, and the kernel crashes. Do the insn in the copy too, and push the copy to the next insn. All other code that pushes a state does the same. No verdict changes in selftests, Cilium and scx. Each fork processes one insn less, and the sequence in which the verifier explores states changes slightly. 62 of 161 progs of Cilium process a different number of insns, from -0.7% to +3.3%. The total is 986970 -> 987013. 2 of 563 progs of scx process one insn less. Fixes: 6ccf49f79327 ("bpf: Use SCEV to widen bounded loops") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-17-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8139e41ace00..02b2915f0014 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17289,11 +17289,14 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, } } +static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct bpf_reg_state *dst_reg, struct bpf_reg_state src_reg); + static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct bpf_reg_state *dst_reg) + struct bpf_reg_state *dst_reg, const struct bpf_reg_state *src_reg) { struct bpf_verifier_state *branch; - struct bpf_reg_state *regs; + struct bpf_reg_state *reg; bool alu32; if (reg_smin(dst_reg) == -1 && reg_smax(dst_reg) == 0) @@ -17303,19 +17306,25 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins else return 0; - branch = push_stack(env, env->insn_idx, env->insn_idx, false); + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) return PTR_ERR(branch); - regs = branch->frame[branch->curframe]->regs; + reg = &branch->frame[branch->curframe]->regs[insn->dst_reg]; if (alu32) { - __mark_reg32_known(®s[insn->dst_reg], 0); + __mark_reg32_known(reg, 0); __mark_reg32_known(dst_reg, -1ull); } else { - __mark_reg_known(®s[insn->dst_reg], 0); + __mark_reg_known(reg, 0); __mark_reg_known(dst_reg, -1ull); } - return 0; + /* + * The insn is done with in the branch too, and what gets to the next insn is a state + * like any other. At this insn the branch is a part of the state that is looked at: + * it must not be compared with it. + */ + clear_scalar_id(reg); + return adjust_scalar_min_max_vals(env, insn, reg, *src_reg); } /* WARNING: This function does calculations on 64-bit values, but the actual @@ -17420,7 +17429,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, break; case BPF_AND: if (tnum_is_const(src_reg.var_off)) { - ret = maybe_fork_scalars(env, insn, dst_reg); + ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg); if (ret) return ret; } @@ -17431,7 +17440,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, break; case BPF_OR: if (tnum_is_const(src_reg.var_off)) { - ret = maybe_fork_scalars(env, insn, dst_reg); + ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg); if (ret) return ret; } From df9e4a87a8d300c371ca31b97999b136c7025b05 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:12 +0000 Subject: [PATCH 44/48] selftests/bpf: Test maybe_fork_scalars() at the head of a counted loop The insn that forks is the head of a loop that scalar evolution counted. The state with 0 that maybe_fork_scalars() splits off is within the state that is kept at the head. The access is out of bounds only in the state with 0. Without the previous patch the verifier accepts the prog. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-18-alexei.starovoitov@gmail.com --- .../selftests/bpf/progs/verifier_bounds.c | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index df8d5309657e..05efa9f4a522 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -2159,6 +2159,47 @@ l0_%=: r0 = 0; \ : __clobber_all); } +/* + * The insn that forks is the head of a loop that scalar evolution counted. The state with + * [-1, 0] is kept there, and the state with 0 that is split off is within it. + */ +SEC("socket") +__description("maybe_fork_scalars: AND at the head of a counted loop rejects OOB") +__failure __msg("invalid access to map value") +__naked void and_scalar_fork_at_long_loop_head_rejects_oob(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r9 = r0; \ + r7 = 0; \ + r6 = -1; \ +l1_%=: r6 &= 8; \ + /* r6 is 8 (current) or 0 (pushed) */ \ + r6 ^= 8; \ + /* pushed path: r6 = 8, OOB for value_size=8 */ \ + r1 = r9; \ + r1 += r6; \ + r0 = *(u8*)(r1 + 0); \ + r1 = *(u64*)(r9 + 0); \ + r6 = -1; \ + if r1 == 0 goto l2_%=; \ + r6 = 0; \ +l2_%=: r7 += 1; \ + if r7 < 100 goto l1_%=; \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + /* * Last jump can be detected as always taken because the intersection of R5 and * R7 32bit tnums produces a constant that isn't within R7's s32 bounds. From 4d0f560f4926417f010ce38f8628c41bcad29c9f Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:13 +0000 Subject: [PATCH 45/48] bpf: Keep stack arguments in arch_bpf_timed_may_goto() The JITs pass the first stack arguments of a call in registers: x86-64 r9 argument 6 arm64 x5-x7 arguments 6-8 riscv64 a5-a7 arguments 6-8 '*(u64 *)(r11 - 8) = rX' writes such a register in the caller. 'rX = *(u64 *)(r11 + 8)' reads it in the callee. A call changes these registers. Thus the verifier accepts no call between the write and the call of the callee, and no call in the callee before the read. may_goto is not a call for the verifier. But one time in 65535 iterations it calls arch_bpf_timed_may_goto(), and this function calls C code. It saves R0-R5 only. *(u64 *)(r11 - 8) = 0 ; the verifier: argument 6 is 0 call sub sub: 1: may_goto 2f ; C code changes r9 goto 1b 2: r2 = *(u64 *)(r11 + 8) ; r2 is a kernel address r1 = map_value r1 += r2 *(u64 *)(r1 + 0) = 0 On x86-64 the C code changes r9 when the time of may_goto is up. With the hpet clocksource a look at the clock changes r9 too. The prog above crashes the kernel. LLVM does not emit such code: it reads stack arguments at the entry of a function, and it writes them immediately before the call. Save and restore these registers too. The cost is 2 insns on x86-64, 4 on arm64 and 6 on riscv64, one time in 65535 iterations. Fixes: 324c3ca6eed6 ("bpf,x86: Implement JIT support for stack arguments") Fixes: 235b2fe772f5 ("bpf, arm64: Add JIT support for stack arguments") Fixes: ba754185f902 ("riscv, bpf: Add BPF stack arguments support for RV64 JIT") Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-19-alexei.starovoitov@gmail.com --- arch/arm64/net/bpf_timed_may_goto.S | 10 ++++++++-- arch/riscv/net/bpf_timed_may_goto.S | 27 ++++++++++++++++++--------- arch/x86/net/bpf_timed_may_goto.S | 3 +++ 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/arch/arm64/net/bpf_timed_may_goto.S b/arch/arm64/net/bpf_timed_may_goto.S index a9a802711a7f..1b4a4f91ffc9 100644 --- a/arch/arm64/net/bpf_timed_may_goto.S +++ b/arch/arm64/net/bpf_timed_may_goto.S @@ -5,13 +5,16 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* Allocate stack space and emit frame record */ - stp x29, x30, [sp, #-64]! + stp x29, x30, [sp, #-96]! mov x29, sp /* Save BPF registers R0 - R5 (x8, x0-x4)*/ stp x8, x0, [sp, #16] stp x1, x2, [sp, #32] stp x3, x4, [sp, #48] + /* Save stack arguments 6 - 8 (x5-x7) */ + stp x5, x6, [sp, #64] + str x7, [sp, #80] /* * Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP @@ -32,9 +35,12 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) ldp x8, x0, [sp, #16] ldp x1, x2, [sp, #32] ldp x3, x4, [sp, #48] + /* Restore stack arguments 6 - 8 (x5-x7) */ + ldp x5, x6, [sp, #64] + ldr x7, [sp, #80] /* Restore FP and LR */ - ldp x29, x30, [sp], #64 + ldp x29, x30, [sp], #96 ret SYM_FUNC_END(arch_bpf_timed_may_goto) diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index fcf850f244c9..6ec37f634aff 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -10,15 +10,16 @@ * - output: updated count in BPF_REG_AX (t0) * * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved - * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) and stack + * arguments 6-8 (a5-a7) are saved across the call; BPF_REG_FP (s5) is + * callee-saved and needs no saving. */ SYM_FUNC_START(arch_bpf_timed_may_goto) - addi sp, sp, -(8*SZREG) - REG_S ra, 7*SZREG(sp) - REG_S s0, 6*SZREG(sp) - addi s0, sp, 8*SZREG + addi sp, sp, -(12*SZREG) + REG_S ra, 11*SZREG(sp) + REG_S s0, 10*SZREG(sp) + addi s0, sp, 12*SZREG /* Save BPF registers R0-R5 (t6, a0-a4) */ REG_S t6, 5*SZREG(sp) @@ -27,6 +28,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S a2, 2*SZREG(sp) REG_S a3, 1*SZREG(sp) REG_S a4, 0*SZREG(sp) + /* Save stack arguments 6-8 (a5-a7) */ + REG_S a5, 6*SZREG(sp) + REG_S a6, 7*SZREG(sp) + REG_S a7, 8*SZREG(sp) add a0, t0, s5 call bpf_check_timed_may_goto @@ -39,9 +44,13 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_L a1, 3*SZREG(sp) REG_L a0, 4*SZREG(sp) REG_L t6, 5*SZREG(sp) + /* Restore stack arguments 6-8 */ + REG_L a5, 6*SZREG(sp) + REG_L a6, 7*SZREG(sp) + REG_L a7, 8*SZREG(sp) - REG_L s0, 6*SZREG(sp) - REG_L ra, 7*SZREG(sp) - addi sp, sp, 8*SZREG + REG_L s0, 10*SZREG(sp) + REG_L ra, 11*SZREG(sp) + addi sp, sp, 12*SZREG ret SYM_FUNC_END(arch_bpf_timed_may_goto) diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index 6c6e03f725a3..dd2dde89836b 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -22,6 +22,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) pushq %rdx pushq %rcx pushq %r8 + /* Save stack argument 6. */ + pushq %r9 /* * r10 has the pointer to count and timestamp. JIT added the frame @@ -36,6 +38,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* BPF_REG_AX=r10 will be stored into count, so move return value to it. */ movq %rax, %r10 + popq %r9 /* Restore r5-r0. */ popq %r8 popq %rcx From b433cdb1cc4f599d840ab33d97f8498625bef629 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Tue, 27 Oct 2026 19:03:14 +0000 Subject: [PATCH 46/48] selftests/bpf: Test stack arguments across may_goto Each test passes arguments 6-8 on the stack. Thus it uses all the registers in which the JITs of x86-64, arm64 and riscv64 pass stack arguments. may_goto then ------------------------------------------- --------------------------- 70000 iterations in the callee the callee reads the args the time is up in the callee the callee reads the args 70000 iterations in the caller, after the the caller calls the callee writes of the args 70000 iterations have one look at the clock. Without the fix on x86-64 the second test fails, and the first test fails with the hpet clocksource. The third test passes there. The result depends on the registers that the C code uses. Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261027190314.1440199-20-alexei.starovoitov@gmail.com --- .../selftests/bpf/progs/verifier_stack_arg.c | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index b9f799f6e405..15a12497e074 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -433,6 +433,125 @@ __naked void stack_arg_sequential_calls(void) ); } +/* + * A JIT passes the first stack arguments in registers. may_goto is not a call: they must be + * the same after it, also when it looks at the clock. That is once in 65535 iterations. + */ +__naked __noinline __used +static long loops_then_reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "r1 = 0;" + "1:" + "may_goto 2f;" + "r1 += 1;" + "if r1 < 70000 goto 1b;" + "2:" + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: may_goto looks at the clock, then the callee reads its args") +__success __retval(876) +__naked void stack_arg_may_goto_in_callee(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "call loops_then_reads_8args;" + "exit;" + ::: __clobber_all + ); +} + +__naked __noinline __used +static long spins_then_reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: the time of may_goto is up, then the callee reads its args") +__success __retval(876) +__naked void stack_arg_may_goto_timeout_in_callee(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "call spins_then_reads_8args;" + "exit;" + ::: __clobber_all + ); +} + +__naked __noinline __used +static long reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: may_goto looks at the clock between the stores of the args and the call") +__success __retval(876) +__naked void stack_arg_may_goto_in_caller(void) +{ + asm volatile ( + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "r6 = 0;" + "1:" + "may_goto 2f;" + "r6 += 1;" + "if r6 < 70000 goto 1b;" + "2:" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call reads_8args;" + "exit;" + ::: __clobber_all + ); +} + #else SEC("socket") From 6c476d2b73056ed151c3f8b884c627c75999ed7c Mon Sep 17 00:00:00 2001 From: Feng Yang Date: Wed, 28 Oct 2026 11:19:33 +0800 Subject: [PATCH 47/48] bpf: Fix out-of-bounds btf->types write for split BTF btf_add_type() stores the types of a BTF in btf->types[], which can hold at most BTF_MAX_TYPE entries. The types of a split BTF get IDs start_id .. start_id + nr_types - 1, so only BTF_MAX_TYPE - start_id of them fit, but neither the limit check nor the size used to grow the array accounts for start_id: the check is an equality test against BTF_MAX_TYPE ('start_id + btf->types_size == BTF_MAX_TYPE') and the new size is clamped to BTF_MAX_TYPE. The check therefore only fires if the array happens to grow to exactly BTF_MAX_TYPE - start_id entries. Otherwise, if the module has more than BTF_MAX_TYPE types, the array is filled up to BTF_MAX_TYPE entries. The next type that needs to be added reallocates the array to the same size, after which btf->types_size no longer equals btf->nr_types and the growth path is not taken anymore, so every following type is written past the end of the array: BUG: unable to handle page fault for address: ffffc90006200000 RIP: 0010:btf_add_type+0x74/0x270 Call Trace: btf_check_all_metas btf_parse_module Bound both the array size and the limit check by BTF_MAX_TYPE - start_id. Base BTF (start_id == 0) is unaffected. Fixes: 951bb64621b8 ("bpf: Add in-kernel split BTF support") Signed-off-by: Feng Yang Signed-off-by: Alexei Starovoitov Reviewed-by: Mahe Tardy Acked-by: Ihor Solodrai Link: https://patch.msgid.link/20261028031933.277550-1-yangfeng59949@163.com --- kernel/bpf/btf.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7ed1ff71a089..1be30487a2fa 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1752,15 +1752,16 @@ static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t) /* Expand 'types' array */ struct btf_type **new_types; - u32 expand_by, new_size; + u32 expand_by, new_size, max_size; - if (btf->start_id + btf->types_size == BTF_MAX_TYPE) { + if (btf->start_id + btf->types_size >= BTF_MAX_TYPE) { btf_verifier_log(env, "Exceeded max num of types"); return -E2BIG; } + max_size = BTF_MAX_TYPE - btf->start_id; expand_by = max_t(u32, btf->types_size >> 2, 16); - new_size = min_t(u32, BTF_MAX_TYPE, + new_size = min_t(u32, max_size, btf->types_size + expand_by); new_types = kvzalloc_objs(*new_types, new_size, From c8a8a68d3ab4a06c8f5a13a8c00275351c8239b4 Mon Sep 17 00:00:00 2001 From: Alexei Starovoitov Date: Fri, 23 Oct 2026 15:03:02 +0000 Subject: [PATCH 48/48] bpf: Don't re-sort subprog_info[] for every new subprogram add_subprog() appends new subprog to env->subprog_info[] and calls sort() on the whole array, though only the last entry can be out of order. Move subprogs that start after the new one up by one entry instead. 'start' is unique, so the order of the array doesn't change. The new entry can land in a used slot, so initialize all of it. add_subprog() returns the index of the new subprog instead of the last one. The callers check for error only. It's a trivial prep for rust-bpf [1] that bumps BPF_MAX_SUBPROGS to 1024. Rust crates have more than 256 functions that are not inlined. Time in add_subprogs() when veristat loads sched_ext schedulers from scx repo, 8 cpu VM, usec: subprogs before after arena_selftest 101 386 27 lavd_enqueue 52 96 16 lavd_dispatch 48 78 11 [1] https://github.com/4ast/rust-bpf Signed-off-by: Alexei Starovoitov Reviewed-by: Mahe Tardy Link: https://patch.msgid.link/20261023150302.4097865-1-alexei.starovoitov@gmail.com --- kernel/bpf/verifier.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 02b2915f0014..26a5ee0708a3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2617,12 +2617,6 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, return &elem->st; } -static int cmp_subprogs(const void *a, const void *b) -{ - return ((struct bpf_subprog_info *)a)->start - - ((struct bpf_subprog_info *)b)->start; -} - /* Find subprogram that contains instruction at 'off' */ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off) { @@ -2657,26 +2651,29 @@ int bpf_find_subprog(struct bpf_verifier_env *env, int off) static int add_subprog(struct bpf_verifier_env *env, int off) { + struct bpf_subprog_info *subprog = env->subprog_info; int insn_cnt = env->prog->len; - int ret; + int i; if (off >= insn_cnt || off < 0) { verbose(env, "call to invalid destination\n"); return -EINVAL; } - ret = bpf_find_subprog(env, off); - if (ret >= 0) - return ret; + i = bpf_find_subprog(env, off); + if (i >= 0) + return i; if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { verbose(env, "too many subprograms\n"); return -E2BIG; } - /* determine subprog starts. The end is one before the next starts */ - env->subprog_info[env->subprog_cnt].exit_idx = U32_MAX; - env->subprog_info[env->subprog_cnt++].start = off; - sort(env->subprog_info, env->subprog_cnt, - sizeof(env->subprog_info[0]), cmp_subprogs, NULL); - return env->subprog_cnt - 1; + /* + * Determine subprog starts. The end is one before the next starts. + * The array stays sorted: subprogs that start after 'off' move up. + */ + for (i = env->subprog_cnt++; i > 0 && subprog[i - 1].start > off; i--) + subprog[i] = subprog[i - 1]; + subprog[i] = (struct bpf_subprog_info){ .start = off, .exit_idx = U32_MAX }; + return i; } static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env)