From 92863e678070f57c17c868e4bfa2441a5c61ad2b Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 23:15:34 +0200 Subject: [PATCH 1/4] bpf: Fix vmlinux BTF prep race in bpf_get_btf_vmlinux bpf_get_btf_vmlinux() lazily parses the vmlinux BTF under the bpf_verifier_lock, but publishes the result through a plain store and re-checks it through a plain lockless load. Nothing orders the stores initializing the struct btf inside btf_parse_vmlinux() against the store publishing the pointer: On a weakly ordered arch, a concurrent first-time caller taking the lockless fast path could in principle observe the pointer before the parsed contents are visible. The mutex_unlock() does not help such a reader given it only synchronizes with a later acquisition of the same lock. Thus, publish the pointer with smp_store_release() and read it on the fast path with smp_load_acquire(). Acquire semantics are needed rather than a dependency-ordered READ_ONCE(): btf_parse_vmlinux() also populates globals outside the returned object (e.g. bpf_ctx_convert.t). An address dependency would only order accesses performed through the pointer and not cover other globals. Fixes: 8580ac9404f6 ("bpf: Process in-kernel BTF") Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708211537.371874-2-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 001ac53825da..9217e0f87cb5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -19559,13 +19559,25 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct btf *bpf_get_btf_vmlinux(void) { - if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { + /* Pairs with the smp_store_release() on the parse path below. */ + struct btf *btf = smp_load_acquire(&btf_vmlinux); + + if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { mutex_lock(&bpf_verifier_lock); - if (!btf_vmlinux) - btf_vmlinux = btf_parse_vmlinux(); + btf = btf_vmlinux; + if (!btf) { + btf = btf_parse_vmlinux(); + /* + * Order the parsed BTF contents and the globals the + * parse populated (e.g. bpf_ctx_convert.t) before + * the pointer publication. Pairs with the acquire + * on the lockless fast path above. + */ + smp_store_release(&btf_vmlinux, btf); + } mutex_unlock(&bpf_verifier_lock); } - return btf_vmlinux; + return btf; } /* From 5e5e94d87dea92cc2e2fadaf3be84771509a86ca Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 23:15:35 +0200 Subject: [PATCH 2/4] bpf: Give vmlinux BTF init its own mutex bpf_get_btf_vmlinux() serializes the lazy vmlinux BTF parse with bpf_verifier_lock, the same mutex bpf_check() holds across the whole verification of an unprivileged program (if enabled; it's disabled by default). The latter can potentially stall the mutex holder for a long time (e.g. via userfaultfd), and therefore block first-time bpf_get_btf_vmlinux() caller from any context, including privileged program loads. Give the vmlinux BTF initialization a dedicated btf_vmlinux_lock so it is independent of the unprivileged verification mutex. The parse only needs mutual exclusion against itself. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708211537.371874-3-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index dff5c0d91641..8c04c340f499 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6451,7 +6451,7 @@ struct btf *btf_parse_vmlinux(void) if (IS_ERR(btf)) goto err_out; - /* btf_parse_vmlinux() runs under bpf_verifier_lock */ + /* btf_parse_vmlinux() runs under btf_vmlinux_lock */ bpf_ctx_convert.t = btf_type_by_id(btf, bpf_ctx_convert_btf_id[0]); err = btf_alloc_id(btf); if (err) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9217e0f87cb5..40e20dfa3212 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -324,6 +324,7 @@ static const char *btf_type_name(const struct btf *btf, u32 id) } static DEFINE_MUTEX(bpf_verifier_lock); +static DEFINE_MUTEX(btf_vmlinux_lock); static DEFINE_MUTEX(bpf_percpu_ma_lock); __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) @@ -19563,7 +19564,7 @@ struct btf *bpf_get_btf_vmlinux(void) struct btf *btf = smp_load_acquire(&btf_vmlinux); if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { - mutex_lock(&bpf_verifier_lock); + mutex_lock(&btf_vmlinux_lock); btf = btf_vmlinux; if (!btf) { btf = btf_parse_vmlinux(); @@ -19575,7 +19576,7 @@ struct btf *bpf_get_btf_vmlinux(void) */ smp_store_release(&btf_vmlinux, btf); } - mutex_unlock(&bpf_verifier_lock); + mutex_unlock(&btf_vmlinux_lock); } return btf; } @@ -20089,7 +20090,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, bpf_get_btf_vmlinux(); - /* grab the mutex to protect few globals used by verifier */ + /* Serialize verification of unprivileged programs. */ if (!is_priv) mutex_lock(&bpf_verifier_lock); From 42560699a83db261d1a671a5eadade460d0f9eee Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 23:15:36 +0200 Subject: [PATCH 3/4] bpf: Account insn_aux_data allocation in bpf_check The insn_aux_data array is allocated with a plain vzalloc(), while every other allocation scoped to the verification - verifier states, explored states, the cfg/scc arrays, liveness masks, jump history - is charged to the loader's memcg via GFP_KERNEL_ACCOUNT. At 136 bytes per instruction it is one of the largest verification-time buffers, in the range of ~130MB for a program at the 1M instruction limit (worst case), and it lives across the whole verification. The buffer is also inconsistent with itself: when instruction patching grows it, the vrealloc() in bpf_patch_insn_data() already passes GFP_KERNEL_ACCOUNT. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708211537.371874-4-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 40e20dfa3212..ad8ff228c963 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -20096,7 +20096,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, len = env->prog->len; env->insn_aux_data = - vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); + __vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len), + GFP_KERNEL_ACCOUNT | __GFP_ZERO); ret = -ENOMEM; if (!env->insn_aux_data) goto skip_full_check; From ff755b6007908730946c155bb0d90ebc55926da7 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Wed, 8 Jul 2026 23:15:37 +0200 Subject: [PATCH 4/4] bpf: Account scratch buffer in bpf_prog_calc_tag bpf_prog_calc_tag() copies the instructions into a plain vmalloc() scratch buffer to blind the map fds before hashing. The buffer scales with the program, up to ~8MB at the 1M instruction limit, and is allocated on every program load, but unlike the rest of the load-time scratch memory it is not charged to the loader's memcg. Use GFP_KERNEL_ACCOUNT to account it like the other allocations scoped to the verification/load. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260708211537.371874-5-daniel@iogearbox.net Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 6e19a030da6f..f2b6e4c888af 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -305,7 +305,7 @@ int bpf_prog_calc_tag(struct bpf_prog *fp) bool was_ld_map; u32 i; - dst = vmalloc(size); + dst = __vmalloc(size, GFP_KERNEL_ACCOUNT); if (!dst) return -ENOMEM;