From aed1bf1a352acd4f5ffd2a81e23d2bb6e8e9b636 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 19 Aug 2026 19:20:21 -1000 Subject: [PATCH 01/90] bpf: sched_ext: Mark ops argument container pointer fields as trusted Walking an unannotated pointer field of a trusted struct yields a bare PTR_TO_BTF_ID in non-sleepable programs, which kfuncs and helpers accept, but PTR_UNTRUSTED in sleepable programs, which they reject. This gets in the way of making ops.init_task() sleepable, which schedulers want for allocations. For example, passing args->cgroup into bpf_cgrp_storage_get() then fails verification and the only recourse is round-tripping through the cgroup ID with bpf_cgroup_from_id(). The pointer fields in the sched_ext ops argument containers are all pinned by the callers for the duration of the ops calls and are never NULL. Add them to the verifier's trusted-fields whitelist so that they are PTR_TRUSTED in both sleepable and non-sleepable programs. Fixes: 8195136669661 ("sched_ext: Add cgroup support") Signed-off-by: Tejun Heo Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/95d7ccc17681aa3a4a2eeb1b073f00f7@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e421ea2b80c3..821b47ac75c5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5877,6 +5877,28 @@ BTF_TYPE_SAFE_TRUSTED(struct file) { struct inode *f_inode; }; +/* + * The pointer fields in the sched_ext ops argument containers are pinned by the + * callers for the duration of the ops calls and are never NULL. + */ +BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) { +#ifdef CONFIG_EXT_GROUP_SCHED + struct cgroup *cgroup; +#endif +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) { + struct task_struct *task; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) { + struct sched_ext_ops *ops; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) { + struct sched_ext_ops *ops; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode; }; @@ -5921,6 +5943,10 @@ static bool type_is_trusted(struct bpf_verifier_env *env, BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); } From ee66e3796b80b8d95430ad5d765b4ab24be06d52 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:11 -0400 Subject: [PATCH 02/90] selftests/bpf: libarena: Normalize SPDX headers across files Normalize SPDX in libarena for missing/inconsistent headers. There are currently three files missing headers and two of them (both originally authored by Kartikeya, CC'ed below) that are GPLv2 only. This patch ensures all files in libarena have a dual GPLv2/BSD-2-Clause license in the example of other non- kernel BPF codebases, e.g., libbpf. Add the customary BPF selftests/ SPDX tag to the bpf_may_goto.h file that holds the may_goto and can_loop macros for BPF code. The original authors of the code from when it was still in bpf_experimental.h are CC'ed below. Also add the missing SPDX license to the libarena bitmap.h header. For bpf_arena_spinlock.h and bpf_atomic.h, move the license to Dual GPLv2/BSD-2-Clause. This ensures all components have the same license, with the same rationale as libbpf in tools/lib. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-2-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/libarena/include/bpf_arena_spin_lock.h | 2 +- tools/testing/selftests/bpf/libarena/include/bpf_atomic.h | 2 +- tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h | 1 + tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h | 1 + .../testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c | 3 +++ 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h index 71d9db610263..055b5892cb63 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_arena_spin_lock.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #ifndef BPF_ARENA_SPIN_LOCK_H #define BPF_ARENA_SPIN_LOCK_H diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h index 43c306e17f19..65582b2010ad 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_atomic.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */ #ifndef BPF_ATOMIC_H #define BPF_ATOMIC_H diff --git a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h index 9ba90689d6ba..b32a420d4e1a 100644 --- a/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h +++ b/tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once /* diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h index e2431ea6fdd6..8c5936ae9958 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once #define BITS_PER_BYTE 8 diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c index 76319a529f02..e66b3a26ca3d 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_bitmap.bpf.c @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + #include #include From db00d61fa433d40fe444d8887d143b895372b274 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:12 -0400 Subject: [PATCH 03/90] selftests/bpf: libarena: Inline nonatomic bitmap operations The libarena code currently defines the non-atomic bitmap set/get operations as __weak functions, in accordance with the libarena coding style. This, however, is significant overhead to call functions that span single-digit instructions. Make an exception and expose the getters/setters as static inline functions in the header. Since the function body is now inlined into the caller, mark reads/writes with READ_ONCE()/WRITE_ONCE() to prevent compiler optimizations from breaking code that locklessly polls the bitmap. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-3-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/libarena/include/libarena/bitmap.h | 29 +++++++++++++++++-- .../selftests/bpf/libarena/src/bitmap.bpf.c | 18 ------------ 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h index 8c5936ae9958..163e2b83d943 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/bitmap.h @@ -1,6 +1,8 @@ // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause #pragma once +#include + #define BITS_PER_BYTE 8 #define BYTES_TO_BITS(nb) ((nb) * BITS_PER_BYTE) @@ -16,11 +18,8 @@ struct arena_bitmap { struct arena_bitmap __arena *bmp_alloc(size_t bits); void bmp_free(struct arena_bitmap __arena *bmp); -void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); -void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); void bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp); void bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); -bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp); bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp); bool bmp_test_and_set_bit(u32 bit, struct arena_bitmap __arena *bmp); @@ -33,3 +32,27 @@ void bmp_copy(size_t bits, struct arena_bitmap __arena *dst, struct arena_bitmap bool bmp_intersects(size_t bits, struct arena_bitmap __arena *arg1, struct arena_bitmap __arena *arg2); bool bmp_subset(size_t bits, struct arena_bitmap __arena *big, struct arena_bitmap __arena *small); void bmp_print(size_t bits, struct arena_bitmap __arena *bmp); + +static __always_inline +void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + volatile u64 __arena *word = &bmp->bits[BIT_WORD(bit)]; + + *word |= BIT_MASK(bit); +} + +static __always_inline +void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + volatile u64 __arena *word = &bmp->bits[BIT_WORD(bit)]; + + *word &= ~BIT_MASK(bit); +} + +static __always_inline +bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) +{ + u64 word = READ_ONCE(bmp->bits[BIT_WORD(bit)]); + + return word & BIT_MASK(bit); +} diff --git a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c index 5ff8e688ddc7..0390f20ce366 100644 --- a/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/bitmap.bpf.c @@ -34,24 +34,6 @@ void bmp_free(struct arena_bitmap __arena *bmp) arena_free(bmp); } -__weak -void __bmp_set_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - bmp->bits[BIT_WORD(bit)] |= BIT_MASK(bit); -} - -__weak -void __bmp_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - bmp->bits[BIT_WORD(bit)] &= ~BIT_MASK(bit); -} - -__weak -bool bmp_test_bit(u32 bit, struct arena_bitmap __arena *bmp) -{ - return bmp->bits[BIT_WORD(bit)] & BIT_MASK(bit); -} - __weak bool bmp_test_and_clear_bit(u32 bit, struct arena_bitmap __arena *bmp) { From 620e1e2b4377f03834894f86ac85ec865983ad0a Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:13 -0400 Subject: [PATCH 04/90] selftests/bpf: libarena: Disable IRQs during allocation The libarena buddy allocator currently uses arena_spin_lock/unlock to protect its internal data structures in its critical section. These locks disable preemption, but not IRQs. This in turns can cause ABBA deadlocks when an allocation/free operation gets an IRQ while in the critical section, and can only resume after another operation that in turn blocks on the buddy lock. We have concretely seen this with sched_ext schedulers: a) Task 1 on CPU A attempts an allocation during initialization, which is done without holding an rq lock. The task takes an IRQ in the middle of the allocation. b) Task 2 on CPU B exits. It attempts to take the buddy allocator lock during its sched-ext state teardown, and blocks on the spinlock. It does so while holding CPU B's rq lock. c) The scheduler run on CPU A and attempts to move tasks from CPU B's rq to CPU A's rq before resuming running Task 1. This requires B's rq lock, which requires Task 2 to take the buddy allocator lock first. Fix this by disabling IRQs when taking the buddy lock. We use the already existing arena_spin_[lock_irqsave, unlock_irqrestore] calls for this. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-4-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../selftests/bpf/libarena/src/buddy.bpf.c | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c index c674ee5cfcc1..2490ab1396de 100644 --- a/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/buddy.bpf.c @@ -5,6 +5,8 @@ #include #include +#include + /* * Buddy allocator arena-based implementation. * @@ -45,15 +47,8 @@ enum { BUDDY_CHUNK_PAGES = BUDDY_CHUNK_BYTES / __PAGE_SIZE }; -static inline int buddy_lock(struct buddy __arena *buddy) -{ - return arena_spin_lock(&buddy->lock); -} - -static inline void buddy_unlock(struct buddy __arena *buddy) -{ - arena_spin_unlock(&buddy->lock); -} +#define buddy_lock(buddy, flags) (arena_spin_lock_irqsave(&(buddy)->lock, (flags))) +#define buddy_unlock(buddy, flags) (arena_spin_unlock_irqrestore(&(buddy)->lock, (flags))) /* * Reserve part of the arena address space for the allocator. We use @@ -385,6 +380,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) { u64 order, ord, min_order, max_order; struct buddy_chunk __arena *chunk; + unsigned long flags; size_t left; int power2; u64 vaddr; @@ -416,7 +412,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) return NULL; } - if (buddy_lock(buddy)) { + if (buddy_lock(buddy, flags)) { /* * We cannot reclaim the vaddr space, but that is ok - this * operation should always succeed. The error path is to catch @@ -520,7 +516,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) arena_stderr( "chunk has size of 0x%lx bytes (left %lx bytes)\n", sizeof(*chunk), left); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -531,7 +527,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) order = (power2 >= BUDDY_MIN_ALLOC_SHIFT) ? power2 - BUDDY_MIN_ALLOC_SHIFT : 0; if (idx_set_allocated(chunk, idx, true)) { - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -547,7 +543,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) */ min_order = left ? order + 1 : order; if (add_leftovers_to_freelist(chunk, idx, min_order, max_order)) { - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return NULL; } @@ -556,7 +552,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) max_order = order; } - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return chunk; } @@ -564,6 +560,7 @@ static struct buddy_chunk __arena *buddy_chunk_get(struct buddy __arena *buddy) __weak int buddy_init(struct buddy __arena *buddy) { struct buddy_chunk __arena *chunk; + unsigned long flags; int ret; if (!asan_ready()) @@ -579,7 +576,7 @@ __weak int buddy_init(struct buddy __arena *buddy) chunk = buddy_chunk_get(buddy); - if (buddy_lock(buddy)) { + if (buddy_lock(buddy, flags)) { bpf_arena_free_pages(&arena, chunk, BUDDY_CHUNK_PAGES); return -EINVAL; } @@ -591,7 +588,7 @@ __weak int buddy_init(struct buddy __arena *buddy) /* Put the chunk at the beginning of the list. */ buddy->first_chunk = chunk; - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return chunk ? 0 : -ENOMEM; } @@ -730,9 +727,10 @@ static u64 buddy_alloc_from_existing_chunks(struct buddy __arena *buddy, int ord */ static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_chunk __arena *chunk, int order) { + unsigned long flags; u64 address; - if (buddy_lock(buddy)) + if (buddy_lock(buddy, flags)) return (u64)NULL; @@ -745,7 +743,7 @@ static u64 buddy_alloc_from_new_chunk(struct buddy __arena *buddy, struct buddy_ address = buddy_chunk_alloc(buddy->first_chunk, order); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return (u64)address; } @@ -754,6 +752,7 @@ void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) { void __arena *address = NULL; struct buddy_chunk __arena *chunk; + unsigned long flags; int order; if (!buddy) @@ -765,11 +764,11 @@ void __arena *buddy_alloc(struct buddy __arena *buddy, size_t size) return NULL; } - if (buddy_lock(buddy)) + if (buddy_lock(buddy, flags)) return NULL; address = (u8 __arena *)buddy_alloc_from_existing_chunks(buddy, order); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); if (address) goto done; @@ -880,6 +879,7 @@ static __always_inline int buddy_free_unlocked(struct buddy __arena *buddy, u64 __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) { + unsigned long flags; int ret; if (!buddy) @@ -889,13 +889,13 @@ __weak int buddy_free(struct buddy __arena *buddy, void __arena *addr) if (!addr) return 0; - ret = buddy_lock(buddy); + ret = buddy_lock(buddy, flags); if (ret) return ret; ret = buddy_free_unlocked(buddy, (u64)addr); - buddy_unlock(buddy); + buddy_unlock(buddy, flags); return ret; } From 283cb6513c881ccc6ff81f3da75d55e1ab1fe6fc Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:14 -0400 Subject: [PATCH 05/90] selftests/bpf: libarena: Add calloc() call The arena memory allocator currently does not provide the option to zero-initialize the allocated memory. Provide a calloc() operation that calls memset() on the returned memory. Reuse naive memset implementation already present in the arena ASAN code for now. Subsequent patches will optimize the function. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-5-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/libarena/include/libarena/common.h | 18 +++++++++++++ .../selftests/bpf/libarena/src/asan.bpf.c | 21 ++------------- .../selftests/bpf/libarena/src/common.bpf.c | 27 ++++++++++++++++++- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h index 931ace9a49e2..d32a51ff5e7f 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h @@ -49,6 +49,7 @@ extern volatile u64 asan_violated; int arena_fls(__u64 word); void __arena *arena_malloc(size_t size); +void __arena *arena_calloc(size_t ncount, size_t size); void arena_free(void __arena *ptr); /* @@ -61,6 +62,23 @@ void arena_free(void __arena *ptr); */ #define arena_subprog_init() do { asm volatile ("" :: "r"(&arena)); } while (0) +/* + * BPF does not currently support the memset intrinsics. for large + * sequential copies, or assignments of large data structures, + * the frontend will generate an intrinsic that causes the BPF + * backend to exit due to a missing implementation. Provide + * implementations for the intrinsic. + */ +static inline int arena_memset(s8 __arena *dst, s8 val, size_t size) +{ + size_t i; + + for (i = zero; i < size && can_loop; i++) + dst[i] = val; + + return 0; +} + #else /* ! __BPF__ */ #include diff --git a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c index 5135d5c72a46..de656b69d13a 100644 --- a/tools/testing/selftests/bpf/libarena/src/asan.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/asan.bpf.c @@ -103,23 +103,6 @@ volatile bool asan_inited = false; */ volatile bool asan_report_once = false; -/* - * BPF does not currently support the memset/memcpy/memcmp intrinsics. - * For large sequential copies, or assignments of large data structures, - * the frontend will generate an intrinsic that causes the BPF backend - * to exit due to a missing implementation. Provide a simple implementation - * just for memset to use it for poisoning/unpoisoning the map. - */ -__weak int asan_memset(s8 __arena *dst, s8 val, size_t size) -{ - size_t i; - - for (i = zero; i < size && can_loop; i++) - dst[i] = val; - - return 0; -} - /* Validate a 1-byte access, always within a single byte. */ static __always_inline bool memory_is_poisoned_1(s8 __arena *addr) { @@ -422,7 +405,7 @@ __hidden __noasan int asan_poison(void __arena *addr, s8 val, size_t size) shadow = mem_to_shadow(addr); len = size >> ASAN_SHADOW_SHIFT; - asan_memset(shadow, val, len); + arena_memset(shadow, val, len); return 0; } @@ -463,7 +446,7 @@ __hidden __noasan int asan_unpoison(void __arena *addr, size_t size) shadow = mem_to_shadow(addr); len = size >> ASAN_SHADOW_SHIFT; - asan_memset(shadow, 0, len); + arena_memset(shadow, 0, len); /* * If we are allocating a non-granule aligned region, we need to adjust diff --git a/tools/testing/selftests/bpf/libarena/src/common.bpf.c b/tools/testing/selftests/bpf/libarena/src/common.bpf.c index 41b1de3452fe..d1af60b81838 100644 --- a/tools/testing/selftests/bpf/libarena/src/common.bpf.c +++ b/tools/testing/selftests/bpf/libarena/src/common.bpf.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include + #include #include #include @@ -55,10 +57,33 @@ __weak void __arena *arena_malloc(size_t size) return buddy_alloc(&buddy, size); } +__weak void __arena *arena_calloc(size_t ncount, size_t size) +{ + void __arena *mem; + size_t total; + + /* + * Ideally we'd be using __builtin_mul_overflow here, + * but the BPF compiler backend doesn't implement __multi3. + * There are ways to optimize the division away from the + * overflow check, but any costs are dwarfed by the + * buddy_alloc() call. Keep it simple for now. + */ + if (unlikely(ncount && size >= ULLONG_MAX / ncount)) + return NULL; + + total = ncount * size; + + mem = buddy_alloc(&buddy, total); + if (likely(mem)) + arena_memset(mem, 0, total); + + return mem; +} + __weak void arena_free(void __arena *ptr) { buddy_free(&buddy, ptr); } - char _license[] SEC("license") = "GPL"; From 047099824cbe646c590478550b511530a70969f2 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:15 -0400 Subject: [PATCH 06/90] selftests/bpf: libarena: Add a benchmark for malloc()/calloc() Add a benchmark for measuring the performance of the malloc()/calloc() arena allocator calls. This is useful as a basic allocator performance check that we can easily expand later. As with the regular arena allocator, focus on sub-page allocations that cannot be satisfied efficiently with the BPF arena page allocation code. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-6-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/Makefile | 18 +- tools/testing/selftests/bpf/bench.c | 6 + .../selftests/bpf/benchs/bench_libarena.c | 210 ++++++++++++++++++ .../bpf/benchs/run_bench_libarena.sh | 31 +++ tools/testing/selftests/bpf/libarena/Makefile | 30 ++- .../bpf/libarena/benchs/bench_malloc.bpf.c | 51 +++++ 6 files changed, 334 insertions(+), 12 deletions(-) create mode 100644 tools/testing/selftests/bpf/benchs/bench_libarena.c create mode 100755 tools/testing/selftests/bpf/benchs/run_bench_libarena.sh create mode 100644 tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 2b2f93dec474..5f1a3bfc0569 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -826,18 +826,22 @@ LIBARENA_MAKE_ARGS = \ BPF_TARGET_ENDIAN="$(BPF_TARGET_ENDIAN)" \ Q="$(Q)" -LIBARENA_BPF_DEPS := $(wildcard libarena/Makefile \ - libarena/include/* \ - libarena/include/libarena/* \ - libarena/src/* \ - libarena/selftests/* \ - libarena/*.bpf.o) +LIBARENA_COMMON_DEPS := $(wildcard libarena/Makefile \ + libarena/include/* \ + libarena/include/libarena/* \ + libarena/src/*) +LIBARENA_BPF_DEPS := $(LIBARENA_COMMON_DEPS) $(wildcard libarena/selftests/*) +LIBARENA_BENCH_BPF_DEPS := $(LIBARENA_COMMON_DEPS) $(wildcard libarena/benchs/*) LIBARENA_SKEL := libarena/libarena.skel.h +LIBARENA_BENCH_SKEL := libarena/libarena_bench.skel.h $(LIBARENA_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena.skel.h $(LIBARENA_MAKE_ARGS) +$(LIBARENA_BENCH_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BENCH_BPF_DEPS) | $(LIBARENA_SKEL) + +$(MAKE) -C libarena benchmarks $(LIBARENA_MAKE_ARGS) + ifneq ($(CLANG_HAS_ARENA_ASAN),) LIBARENA_ASAN_SKEL := libarena/libarena_asan.skel.h CFLAGS += -DHAS_BPF_ARENA_ASAN @@ -987,6 +991,7 @@ $(OUTPUT)/bench_sockmap.o: $(OUTPUT)/bench_sockmap_prog.skel.h $(OUTPUT)/bench_lpm_trie_map.o: $(OUTPUT)/lpm_trie_bench.skel.h $(OUTPUT)/lpm_trie_map.skel.h $(OUTPUT)/bench_bpf_nop.o: $(OUTPUT)/bpf_nop_bench.skel.h bench_bpf_timing.h $(OUTPUT)/bench_xdp_lb.o: $(OUTPUT)/xdp_lb_bench.skel.h bench_bpf_timing.h +$(OUTPUT)/bench_libarena.o: $(LIBARENA_BENCH_SKEL) $(OUTPUT)/bench_bpf_timing.o: bench_bpf_timing.h $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ) $(OUTPUT)/bench: LDLIBS += -lm @@ -1014,6 +1019,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \ $(OUTPUT)/bench_bpf_timing.o \ $(OUTPUT)/bench_bpf_nop.o \ $(OUTPUT)/bench_xdp_lb.o \ + $(OUTPUT)/bench_libarena.o \ $(OUTPUT)/usdt_1.o \ $(OUTPUT)/usdt_2.o \ # diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 465233ea5f79..d5f494184f9d 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -288,6 +288,7 @@ extern struct argp bench_crypto_argp; extern struct argp bench_sockmap_argp; extern struct argp bench_lpm_trie_map_argp; extern struct argp bench_xdp_lb_argp; +extern struct argp bench_libarena_argp; static const struct argp_child bench_parsers[] = { { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 }, @@ -306,6 +307,7 @@ static const struct argp_child bench_parsers[] = { { &bench_sockmap_argp, 0, "bpf sockmap benchmark", 0 }, { &bench_lpm_trie_map_argp, 0, "LPM trie map benchmark", 0 }, { &bench_xdp_lb_argp, 0, "XDP load-balancer benchmark", 0 }, + { &bench_libarena_argp, 0, "libarena allocator benchmark", 0 }, {}, }; @@ -585,6 +587,8 @@ extern const struct bench bench_lpm_trie_delete; extern const struct bench bench_lpm_trie_free; extern const struct bench bench_bpf_nop; extern const struct bench bench_xdp_lb; +extern const struct bench bench_libarena_malloc; +extern const struct bench bench_libarena_calloc; static const struct bench *benchs[] = { &bench_count_global, @@ -669,6 +673,8 @@ static const struct bench *benchs[] = { &bench_lpm_trie_free, &bench_bpf_nop, &bench_xdp_lb, + &bench_libarena_malloc, + &bench_libarena_calloc, }; static void find_benchmark(void) diff --git a/tools/testing/selftests/bpf/benchs/bench_libarena.c b/tools/testing/selftests/bpf/benchs/bench_libarena.c new file mode 100644 index 000000000000..24e432244bf5 --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/bench_libarena.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include + +#include "bench.h" + +#include +#include +#include +#include + +#include "libarena/libarena_bench.skel.h" + +static struct { + __u64 alloc_size; + __u64 nallocs; +} args = { + .alloc_size = 64, + .nallocs = 10000, +}; + +static struct { + struct libarena_bench *skel; + int bench_fd; + int reset_fd; +} ctx; + +enum { + ARG_LIBARENA_ALLOC_SIZE = 12000, + ARG_LIBARENA_NALLOCS, +}; + +static const struct argp_option opts[] = { + { "alloc_size", ARG_LIBARENA_ALLOC_SIZE, "BYTES", 0, + "Size of each arena allocation" }, + { "nallocs", ARG_LIBARENA_NALLOCS, "ITERS", 0, + "Number of allocation per measurement" }, + {}, +}; + +static error_t parse_arg(int key, char *arg, struct argp_state *state) +{ + unsigned long value; + + switch (key) { + case ARG_LIBARENA_ALLOC_SIZE: + value = strtoull(arg, NULL, 10); + if (!value || value >= UINT_MAX) { + fprintf(stderr, "invalid alloc_size: %ld", value); + argp_usage(state); + } + args.alloc_size = value; + break; + case ARG_LIBARENA_NALLOCS: + args.nallocs = strtoull(arg, NULL, 10); + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +const struct argp bench_libarena_argp = { + .options = opts, + .parser = parse_arg, +}; + +static void validate(void) +{ + if (env.consumer_cnt != 0) { + fprintf(stderr, "benchmark doesn't support consumers\n"); + exit(1); + } + + if (env.producer_cnt != 1) { + fprintf(stderr, "benchmark supports exactly one producer\n"); + exit(1); + } +} + +static void setup_common(void) +{ + struct arena_alloc_reserve_args reserve_args = { + .nr_pages = ARENA_RESERVE_PAGES_DFL, + }; + int err; + + setup_libbpf(); + + ctx.skel = libarena_bench__open_and_load(); + if (!ctx.skel) { + fprintf(stderr, "failed to open and load skeleton\n"); + exit(1); + } + + err = libarena_run_prog_args( + bpf_program__fd(ctx.skel->progs.arena_alloc_reserve), + &reserve_args, sizeof(reserve_args)); + if (err) { + fprintf(stderr, "failed to reserve arena pages: %d\n", err); + exit(1); + } + + err = libarena_run_prog( + bpf_program__fd(ctx.skel->progs.arena_buddy_reset)); + if (err) { + fprintf(stderr, "failed to initialize arena allocator: %d\n", err); + exit(1); + } + + ctx.skel->bss->bench_alloc_size = args.alloc_size; + ctx.skel->bss->bench_nallocs = args.nallocs; + ctx.reset_fd = bpf_program__fd(ctx.skel->progs.arena_buddy_reset); +} + +static void malloc_setup(void) +{ + setup_common(); + ctx.bench_fd = bpf_program__fd(ctx.skel->progs.bench_malloc); +} + +static void calloc_setup(void) +{ + setup_common(); + ctx.bench_fd = bpf_program__fd(ctx.skel->progs.bench_calloc); +} + +static void *producer(void *input) +{ + int err; + + while (true) { + err = libarena_run_prog(ctx.bench_fd); + if (err) { + fprintf(stderr, "libarena benchmark failed: %d\n", err); + exit(1); + } + + err = libarena_run_prog(ctx.reset_fd); + if (err) { + fprintf(stderr, "libarena alloc reset failed: %d\n", err); + exit(1); + } + } + + return NULL; +} + +static void measure(struct bench_res *res) +{ + res->duration_ns = atomic_swap(&ctx.skel->bss->bench_duration_ns, 0); + res->hits = atomic_swap(&ctx.skel->bss->bench_hits, 0); +} + +static void report_progress(int iter, struct bench_res *res, long delta_ns) +{ + double latency_ns = 0.0; + + if (res->hits) + latency_ns = res->duration_ns / (double)res->hits; + + printf("Iter %3d (%7.3lfus): latency %8.3lf ns/op (%ld allocations)\n", + iter, (delta_ns - 1000000000) / 1000.0, latency_ns, res->hits); +} + +static void report_final(struct bench_res res[], int res_cnt) +{ + unsigned long duration_ns = 0; + long hits = 0; + int i; + + for (i = 0; i < res_cnt; i++) { + duration_ns += res[i].duration_ns; + hits += res[i].hits; + } + + if (!hits || !res_cnt) { + printf("Summary: no runs measured\n"); + return; + } + + printf("Summary: %.3lf ns/op, %.0lf invocations for %u allocations/invocation)\n", + duration_ns / (double)hits, hits / (double)res_cnt, + ctx.skel->bss->bench_nallocs); +} + +const struct bench bench_libarena_malloc = { + .name = "libarena-malloc", + .argp = &bench_libarena_argp, + .validate = validate, + .setup = malloc_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = report_progress, + .report_final = report_final, +}; + +const struct bench bench_libarena_calloc = { + .name = "libarena-calloc", + .argp = &bench_libarena_argp, + .validate = validate, + .setup = calloc_setup, + .producer_thread = producer, + .measure = measure, + .report_progress = report_progress, + .report_final = report_final, +}; diff --git a/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh b/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh new file mode 100755 index 000000000000..10afe4d52ebf --- /dev/null +++ b/tools/testing/selftests/bpf/benchs/run_bench_libarena.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +source ./benchs/run_common.sh + +set -eufo pipefail + +RUN_BENCH="./bench -d3 -q" + +summarize_libarena() +{ + local bench="$1" + local summary + + summary=$(printf '%s\n' "$2" | tail -n1) + summary=${summary#Summary: } + printf "%-20s %s\n" "$bench" "$summary" +} + +header "libarena sequential malloc\n" + +for size in 16 64 256 1024 4096; do +subtitle "allocation size: $size" +printf "\t-------------------\n" + for nallocs in 10 50 100 500 1000 5000 10000; do + summarize_libarena "malloc:" \ + "$($RUN_BENCH --alloc_size "$size" --nallocs "$nallocs" libarena-malloc)" + summarize_libarena "calloc:" \ + "$($RUN_BENCH --alloc_size "$size" --nallocs "$nallocs" libarena-calloc)" + done +done diff --git a/tools/testing/selftests/bpf/libarena/Makefile b/tools/testing/selftests/bpf/libarena/Makefile index 5e2ab514805e..6d62eda34920 100644 --- a/tools/testing/selftests/bpf/libarena/Makefile +++ b/tools/testing/selftests/bpf/libarena/Makefile @@ -27,10 +27,17 @@ BPFDIR=$(abspath $(LIBARENA)/..) INCLUDE_DIR ?= $(BPFDIR)/tools/include LIBBPF_INCLUDE ?= $(INCLUDE_DIR) -# Scan src/ and selftests/ to generate the final binaries -LIBARENA_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) $(wildcard $(LIBARENA)/selftests/*.bpf.c) -LIBARENA_OBJECTS = $(notdir $(LIBARENA_SOURCES:.bpf.c=.bpf.o)) -LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_SOURCES:.bpf.c=_asan.bpf.o)) +# Build selftests and benchmarks into separate BPF objects and skeletons. +LIBARENA_CORE_SOURCES = $(wildcard $(LIBARENA)/src/*.bpf.c) +LIBARENA_TEST_SOURCES = $(wildcard $(LIBARENA)/selftests/*.bpf.c) +LIBARENA_BENCH_SOURCES = $(wildcard $(LIBARENA)/benchs/*.bpf.c) + +LIBARENA_OBJECTS = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=.bpf.o) \ + $(LIBARENA_TEST_SOURCES:.bpf.c=.bpf.o)) +LIBARENA_OBJECTS_ASAN = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=_asan.bpf.o) \ + $(LIBARENA_TEST_SOURCES:.bpf.c=_asan.bpf.o)) +LIBARENA_BENCH_OBJECTS = $(notdir $(LIBARENA_CORE_SOURCES:.bpf.c=.bpf.o) \ + $(LIBARENA_BENCH_SOURCES:.bpf.c=.bpf.o)) INCLUDES = -I$(LIBARENA)/include -I$(BPFDIR) ifneq ($(INCLUDE_DIR),) @@ -58,12 +65,19 @@ override BPF_CFLAGS += $(INCLUDES) CFLAGS = -O2 -no-pie CFLAGS += $(INCLUDES) -vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests -vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests +vpath %.bpf.c $(LIBARENA)/src $(LIBARENA)/selftests $(LIBARENA)/benchs +vpath %.c $(LIBARENA)/src $(LIBARENA)/selftests $(LIBARENA)/benchs skeletons: libarena.skel.h libarena_asan.skel.h .PHONY: skeletons +benchmarks: libarena_bench.skel.h +.PHONY: benchmarks + +libarena_bench.skel.h: libarena_bench.bpf.o + $(call msg,GEN-SKEL,libarena,$@) + $(Q)$(BPFTOOL) gen skeleton $< name "libarena_bench" > $@ + libarena_asan.skel.h: libarena_asan.bpf.o $(call msg,GEN-SKEL,libarena,$@) $(Q)$(BPFTOOL) gen skeleton $< name "libarena_asan" > $@ @@ -80,6 +94,10 @@ libarena.bpf.o: $(LIBARENA_OBJECTS) $(call msg,GEN-OBJ,libarena,$@) $(Q)$(BPFTOOL) gen object $@ $^ +libarena_bench.bpf.o: $(LIBARENA_BENCH_OBJECTS) + $(call msg,GEN-OBJ,libarena,$@) + $(Q)$(BPFTOOL) gen object $@ $^ + %_asan.bpf.o: %.bpf.c $(call msg,CLNG-BPF,libarena,$@) $(Q)$(CLANG) $(BPF_CFLAGS) $(ASAN_FLAGS) -DBPF_ARENA_ASAN $(BPF_TARGET_ENDIAN) -c $< -o $@ diff --git a/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c b/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c new file mode 100644 index 000000000000..e06397dc9a40 --- /dev/null +++ b/tools/testing/selftests/bpf/libarena/benchs/bench_malloc.bpf.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include + +#include +#include + +u32 bench_alloc_size; +u32 bench_nallocs; +long bench_hits; +long bench_duration_ns; + +SEC("syscall") +int bench_malloc(void) +{ + void __arena *mem; + u64 start_ns; + u32 i; + + start_ns = bpf_ktime_get_ns(); + for (i = zero; i < bench_nallocs && can_loop; i++) { + mem = arena_malloc(bench_alloc_size); + if (!mem) + return -ENOMEM; + } + + __sync_add_and_fetch(&bench_duration_ns, + bpf_ktime_get_ns() - start_ns); + __sync_add_and_fetch(&bench_hits, i); + return 0; +} + +SEC("syscall") +int bench_calloc(void) +{ + void __arena *mem; + u64 start_ns; + u32 i; + + start_ns = bpf_ktime_get_ns(); + for (i = zero; i < bench_nallocs && can_loop; i++) { + mem = arena_calloc(1, bench_alloc_size); + if (!mem) + return -ENOMEM; + } + + __sync_add_and_fetch(&bench_duration_ns, + bpf_ktime_get_ns() - start_ns); + __sync_add_and_fetch(&bench_hits, i); + return 0; +} From 8be79805b06e23ebdb7321fe3cdb788e949c984e Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 17 Aug 2026 15:16:16 -0400 Subject: [PATCH 07/90] selftests/bpf: libarena: Optimize and make public arena_memset Clang currently provides no __builtin_{memset, memcpy, memcmp} for its BPF backend. This is especially an issue for arena code that is more likely to do these operations on buffers with user-provided bounds. One example is the arena ASAN implementation that uses memset to update the shadow bitmap. Arena ASAN actually already has a naive implementation of this operation. Another user would be a calloc() call that has to zero the memory it returns. Introduce a more optimized version of the memset() operation for arena memory and make it public to all libarena users. The operation uses word-sized assignments to speed up the function for larger sizes. We expose the function through common.h to allow for inlining from the callers. Signed-off-by: Emil Tsalapatis Link: https://lore.kernel.org/bpf/20260817191616.11071-7-emil@etsalapatis.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/libarena/Makefile | 1 + .../bpf/libarena/include/libarena/common.h | 57 ++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/libarena/Makefile b/tools/testing/selftests/bpf/libarena/Makefile index 6d62eda34920..91164d43bd61 100644 --- a/tools/testing/selftests/bpf/libarena/Makefile +++ b/tools/testing/selftests/bpf/libarena/Makefile @@ -60,6 +60,7 @@ override BPF_CFLAGS += -O2 -g override BPF_CFLAGS += -Wno-incompatible-pointer-types-discards-qualifiers # Required for suppressing harmless vmlinux.h-related warnings. override BPF_CFLAGS += -Wno-missing-declarations +override BPF_CFLAGS += -fno-strict-aliasing override BPF_CFLAGS += $(INCLUDES) CFLAGS = -O2 -no-pie diff --git a/tools/testing/selftests/bpf/libarena/include/libarena/common.h b/tools/testing/selftests/bpf/libarena/include/libarena/common.h index d32a51ff5e7f..a6220355a697 100644 --- a/tools/testing/selftests/bpf/libarena/include/libarena/common.h +++ b/tools/testing/selftests/bpf/libarena/include/libarena/common.h @@ -71,10 +71,63 @@ void arena_free(void __arena *ptr); */ static inline int arena_memset(s8 __arena *dst, s8 val, size_t size) { + size_t headalign; + size_t tailalign; + u8 uval = (u8)val; + size_t val64; size_t i; - for (i = zero; i < size && can_loop; i++) - dst[i] = val; + /* + * Calculate how many bytes to the next word-aligned one. + * We get this by truncating the 2s complement of the + * pointer to the last 3 bits. Intuitively, since + * + * The N LSBs of dst and -dst add to 1 << N, which + * is why dst + (-dst) = 0x0ULL through overflow. So the + * last N = 3 bits of the negative are the number of + * bytes to align dst on the last 3 bits. + * + */ + headalign = -(u64)dst & (sizeof(u64) - 1); + if (!headalign || size < headalign) + goto ptraligned; + + for (i = zero; i < headalign && can_loop; i++) + dst[i] = uval; + + dst += headalign; + size -= headalign; + +ptraligned: + + /* + * Make a word with all bytes equal to the byte we are setting. + * Since 1 byte -> 2 hex digits. + * + * Shifting the value by a 0 bytes is equal to multiplication by 0x01 + * Shifting by 1 bytes is equal to multiplication by 0x01 << 8, + * ... + * Shifting by 7 bytes is equal to multiplication by 0x01 << 56. + * + * End operation to replicate the byte into all the bytes of a word + * is (since a | b = a + b when a & b == 0): + * + * val + val * (1UL << 8) + val * (1UL << 16) + .. + val * (1UL << 56) + * = val * (1UL << 56 + 1UL << 48 + ... + 1UL << 0) + * = val * (0x01UL << 56 | 0x01UL << 48 + ... + 1UL << 0) + * = val * 0x0101 0101 0101 0101 + */ + val64 = (u8)val * 0x0101010101010101ULL; + + /* Pointer is now aligned, use word-aligned assignments. */ + for (i = zero; i < size / sizeof(u64) && can_loop; i++) + ((u64 __arena *)dst)[i] = val64; + + /* Go back to byte-aligned for the tail. */ + tailalign = size % sizeof(u64); + dst += size - tailalign; + for (i = zero; i < tailalign && can_loop; i++) + dst[i] = uval; return 0; } From 4e3fb5bba680c1778e05a120c3793d22204f8fd4 Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:18 +0000 Subject: [PATCH 08/90] bpf: Simplify cnum contains() by removing redundant branches Bearing in mind that a non-empty cnum represents an inclusive circular range in the corresponding unsigned integer domain, membership could be tested by checking whether the distance from the range base to the queried value is within the range size. This makes the explicit wrapping and non-wrapping cases in contains() unnecessary, since: v - cnum.base <= cnum.size is equivalent for both ordinary ranges and ranges that cross the unsigned wrap boundary. We've also run some benchmarks to compare the performance of the new implementation, for example, for empty ranges, overflowing ranges non-overflowing ranges. We've run the tests with O2 optimization. Results: | Test | contains | contains_new | | -------------------- | ----------- | ------------ | | empty | 1.511 ns/op | 1.479 ns/op | | non-overflow inside | 1.156 ns/op | 1.270 ns/op | | non-overflow outside | 1.053 ns/op | 1.258 ns/op | | overflow high side | 1.469 ns/op | 1.257 ns/op | | overflow low side | 1.469 ns/op | 1.270 ns/op | | overflow outside | 1.478 ns/op | 1.256 ns/op | | singleton inside | 1.049 ns/op | 1.256 ns/op | | singleton outside | 1.051 ns/op | 1.254 ns/op | | full range | 1.053 ns/op | 1.256 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves the soundness of the operation are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-2-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index a90e317e3578..ac1f62a0195d 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -210,12 +210,7 @@ bool FN(is_empty)(struct cnum_t cnum) bool FN(contains)(struct cnum_t cnum, ut v) { - if (FN(is_empty)(cnum)) - return false; - if (FN(urange_overflow)(cnum)) - return v >= cnum.base || v <= (ut)cnum.base + cnum.size; - else - return v >= cnum.base && v <= (ut)cnum.base + cnum.size; + return !FN(is_empty)(cnum) && v - cnum.base <= cnum.size; } bool FN(is_const)(struct cnum_t cnum) From b601a4ecbe672371776785bd7ac142af0ac395ac Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:24 +0000 Subject: [PATCH 09/90] bpf: Simplify cnum normalize() by dropping unnecessary comparisons In normalize(), there's no need to compare cnum.base with ST_MAX, because any cnum with size == UT_MAX represents the full unsigned domain independently of base, so normalize() can canonicalize all such values to base 0. Also, it's a no-op assigning 0 to cnum.base if the cnum is already unbounded, so we can remove the base != 0 check too. We've also run some benchmarks to compare the performance of the new implementation. We've run the tests with O2 optimization. Results: | Test | normalize | normalize_new | | ----------------------- | ----------- | ------------- | | normal small range | 1.255 ns/op | 1.217 ns/op | | full range normalized | 1.019 ns/op | 1.217 ns/op | | full range nonzero base | 1.031 ns/op | 1.221 ns/op | | empty | 1.023 ns/op | 1.217 ns/op | | signed max full range | 1.020 ns/op | 1.218 ns/op | | max base singleton | 1.218 ns/op | 1.219 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves soundness of the operation are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-3-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index ac1f62a0195d..09535edbbafc 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -181,7 +181,7 @@ void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max) static inline struct cnum_t FN(normalize)(struct cnum_t cnum) { - if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX) + if (cnum.size == UT_MAX) cnum.base = 0; return cnum; } From 705249119d942e7228d9207800ddfa13e4601b19 Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:32 +0000 Subject: [PATCH 10/90] bpf: Avoid redundant min()/max() in cnum signed bounds Simplify the smin() and smax() calculation by removing the integer min and max calculation on the non-overflowing branch. This is a valid simplification because when the arc does not cross the signed ST_MAX/ST_MIN boundary, the signed order of the endpoints is preserved, so base is the signed minimum and base + size is the signed maximum. Therefore, the min()/max() calls are redundant. We've also run benchmarks to compare the performance of this change. With O2 optimization we have: smin() results: | Test | smin | smin_new | | ------------------------ | ----------- | ----------- | | empty | 1.866 ns/op | 1.477 ns/op | | positive range | 1.870 ns/op | 1.248 ns/op | | negative range | 1.657 ns/op | 1.246 ns/op | | cross zero | 1.870 ns/op | 1.660 ns/op | | signed overflow boundary | 1.463 ns/op | 1.049 ns/op | | signed overflow range | 1.519 ns/op | 1.050 ns/op | | unsigned overflow range | 1.883 ns/op | 1.669 ns/op | | full range | 1.555 ns/op | 1.049 ns/op | smax() results: | Test | smax | smax_new | | ------------------------ | ----------- | ----------- | | empty | 1.462 ns/op | 1.456 ns/op | | positive range | 1.250 ns/op | 1.249 ns/op | | negative range | 1.259 ns/op | 1.297 ns/op | | cross zero | 1.471 ns/op | 1.475 ns/op | | signed overflow boundary | 1.056 ns/op | 1.260 ns/op | | signed overflow range | 1.049 ns/op | 1.259 ns/op | | unsigned overflow range | 1.470 ns/op | 1.477 ns/op | | full range | 1.072 ns/op | 1.263 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves soundness of these operations are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-4-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index 09535edbbafc..30685e43de04 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -33,7 +33,12 @@ struct cnum_t FN(from_srange)(st min, st max) return (struct cnum_t){ .base = base, .size = size }; } -/* True if this cnum represents two unsigned ranges. */ +/* + * True if this cnum represents two unsigned ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(urange_overflow)(struct cnum_t cnum) { /* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */ @@ -44,6 +49,9 @@ static inline bool FN(urange_overflow)(struct cnum_t cnum) * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum. * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range * [0..UT_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_umin / cnum{T}_umax. */ ut FN(umin)(struct cnum_t cnum) { @@ -57,7 +65,12 @@ ut FN(umax)(struct cnum_t cnum) } EXPORT_SYMBOL_GPL(FN(umax)); -/* True if this cnum represents two signed ranges. */ +/* + * True if this cnum represents two signed ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(srange_overflow)(struct cnum_t cnum) { return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN); @@ -67,19 +80,18 @@ static inline bool FN(srange_overflow)(struct cnum_t cnum) * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum. * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range * [ST_MIN..ST_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_smin / cnum{T}_smax. */ st FN(smin)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MIN - : min((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MIN : (st)cnum.base; } st FN(smax)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MAX - : max((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MAX : (st)(cnum.base + cnum.size); } /* From da13c047cda6ef1bbc9355adb040d5b4eecea710 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:44 -0700 Subject: [PATCH 11/90] bpf: Factor check_global_ret_scalar_reg() out of the global return check check_global_subprog_return_code() verifies that a global subprogram returns void, an arena pointer, or register R0 holding a scalar value. Later patches in this series add 16-byte aggregate return support, whose second half is returned in R2 and needs the same validation. Factor the per-register check into check_global_ret_scalar_reg(env, regno) so that it can be reused for R2. No functional change. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055244.3294700-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 821b47ac75c5..5265563c8abf 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17479,37 +17479,45 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char return 0; } -static int check_global_subprog_return_code(struct bpf_verifier_env *env) +static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno) { - struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); - struct bpf_func_state *cur_frame = cur_func(env); + struct bpf_reg_state *reg; int err; - if (subprog_returns_void(env, cur_frame->subprogno)) - return 0; - - err = check_reg_arg(env, BPF_REG_0, SRC_OP); + err = check_reg_arg(env, regno, SRC_OP); if (err) return err; /* Pointers to arena are safe to pass between subprograms. */ - if (is_arena_reg(env, BPF_REG_0)) + if (is_arena_reg(env, regno)) return 0; - if (is_pointer_value(env, BPF_REG_0)) { - verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); + if (is_pointer_value(env, regno)) { + verbose(env, "R%d leaks addr as return value\n", regno); return -EACCES; } + reg = reg_state(env, regno); if (reg->type != SCALAR_VALUE) { - verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", - reg_type_str(env, reg->type)); + verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", + regno, reg_type_str(env, reg->type)); return -EINVAL; } return 0; } +static int check_global_subprog_return_code(struct bpf_verifier_env *env) +{ + struct bpf_func_state *cur_frame = cur_func(env); + u32 subprog = cur_frame->subprogno; + + if (subprog_returns_void(env, subprog)) + return 0; + + return check_global_ret_scalar_reg(env, BPF_REG_0); +} + /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) From cf5b96d9554815780f864d6f0014ac384e9be59c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:49 -0700 Subject: [PATCH 12/90] bpf: Add helpers to describe the R0: R2 return register pair LLVM 23 added support for returning a value in two registers for an __int128, or a struct/union whose size is greater than 8 but not more than 16 bytes: such a value comes back in the R0:R2 register pair, with R2 holding the upper half. See LLVM patches [1] and [2]. Later patches teach the JIT, live register analysis and the verifier itself about that convention. They need to answer the same question: does this subprogram return its value in a register pair? Add bpf_ret_reg_pair() up front so it can be used in subsequent patches. It is answered from a per-subprogram flag that bpf_compute_subprog_ret_regs() derives once from the BTF prototype. jit_requested only says that the JIT is enabled, not that it succeeded: bpf_fixup_call_args() falls back to the interpreter when bpf_jit_subprogs() fails with anything other than -EFAULT. So jit_required is still set once the pair is modelled, which turns that fallback into a load failure rather than a silent divergence from what was verified. [1] https://github.com/llvm/llvm-project/pull/190894 [2] https://github.com/llvm/llvm-project/pull/206876 Signed-off-by: Yonghong Song Link: https://patch.msgid.link/20260819055249.3295283-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 7 ++++ kernel/bpf/verifier.c | 66 ++++++++++++++++++++++++++++++------ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5fad59fdab0d..953d8191563a 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -819,6 +819,8 @@ struct bpf_subprog_info { bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; @@ -1055,6 +1057,11 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { u8 num_params; bool is_void; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5265563c8abf..2be5867d4412 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -385,27 +385,66 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; } -static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog) { - const struct btf_type *type, *func, *func_proto; + const struct btf_type *func, *func_proto; const struct btf *btf = env->prog->aux->btf; u32 btf_id; + if (!btf || !env->prog->aux->func_info) + return NULL; + btf_id = env->prog->aux->func_info[subprog].type_id; + /* Both already validated by prepare_btf_func() at prog load. */ func = btf_type_by_id(btf, btf_id); - if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id)) - return false; - func_proto = btf_type_by_id(btf, func->type); - if (!func_proto) - return false; - type = btf_type_skip_modifiers(btf, func_proto->type, NULL); - if (!type) - return false; + return btf_type_skip_modifiers(btf, func_proto->type, NULL); +} - return btf_type_is_void(type); +static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +{ + const struct btf_type *type = subprog_ret_type(env, subprog); + + return type && btf_type_is_void(type); +} + +static u32 ret_regs_cnt(u32 size) +{ + return size > 8 && size <= 16 ? 2 : 1; +} + +static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) +{ + const struct btf *btf = env->prog->aux->btf; + const struct btf_type *type; + int subprog; + u32 size; + + if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux)) + return 0; + + /* + * Skip the main program: its return value is the program's exit code, + * read out of R0, so it never uses the register pair. An extension does + * have a real prototype for subprog 0, but btf_check_func_type_match() + * refuses to replace a function returning more than 8 bytes. + */ + for (subprog = 1; subprog < env->subprog_cnt; subprog++) { + type = subprog_ret_type(env, subprog); + if (!type || btf_type_is_void(type)) + continue; + if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env, + "cannot size return type of subprog %d", subprog)) + return -EFAULT; + if (ret_regs_cnt(size) > 1) { + subprog_info(env, subprog)->ret_reg_pair = true; + env->prog->jit_required = 1; + } + } + + return 0; } const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) @@ -21185,6 +21224,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* must precede the first bpf_ret_reg_pair() user below */ + ret = bpf_compute_subprog_ret_regs(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; From 34c3503cb29ad0f73d413b83a0586e53d832bb90 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:54 -0700 Subject: [PATCH 13/90] bpf: Wire up JIT support for 16-byte kfunc returns LLVM 23 returns an __int128, or a struct/union larger than 8 bytes and no larger than 16 bytes, in the BPF R0:R2 register pair. The previous patch added the shared helpers describing that convention; wire up the JIT side so that the second half of the return value actually lands in R2. Placing the second return half into R2 is possible on any JIT, but it needs architecture-specific JIT work. Rather than requiring every JIT to implement it at once, add a bpf_jit_supports_kfunc_ret_reg_pair() capability, defaulting to false in the generic core; an architecture opts in once its JIT handles the R0:R2 pair, and the remaining ones are left for future work. Only the x86-64, arm64 and riscv64 JITs opt in so far. On arm64 and riscv64 the native second return register is already BPF R2 (x1 in bpf2a64[] and a1 in regmap[] respectively), so the upper half needs no move at all, unlike x86-64's RDX->RSI. The lower half is covered by the move into BPF R0 that those JITs already emit after every call, from x0 into x8 and from a0 into a5. This has been tested on x86-64 and arm64. The riscv64 path is expected to work by the same register-mapping reasoning as arm64 but has not been tested. bpf_add_kfunc_call() also rejects a kfunc that is marked KF_FASTCALL and returns more than 8 bytes. The bpf_fastcall contract implemented by mark_fastcall_pattern_for_call() assumes a call clobbers R0 plus the registers holding its arguments, so a return in the R0:R2 pair would clobber an R2 the caller expects the fastcall pattern to preserve. Such a kfunc is rejected with -EOPNOTSUPP as well. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055254.3295610-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- arch/arm64/net/bpf_jit_comp.c | 5 +++++ arch/riscv/net/bpf_jit_comp64.c | 5 +++++ arch/x86/net/bpf_jit_comp.c | 27 ++++++++++++++++++++------- include/linux/filter.h | 1 + kernel/bpf/core.c | 5 +++++ kernel/bpf/verifier.c | 12 ++++++++++++ 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c18e005a41db..3aa3ea0bc30b 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2388,6 +2388,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 74efe4b138d2..47c7bf431ba8 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -2121,6 +2121,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 1a9fb530adc3..48429fae0641 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1689,17 +1689,12 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, * arena NULL is offset 0. Return the number of emitted bytes. */ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, - const struct bpf_insn *insn, u8 **pprog) + const struct btf_func_model *fm, u8 **pprog) { - const struct btf_func_model *fm; u8 *prog = *pprog; u8 *start = prog; int i; - fm = bpf_jit_find_kfunc_model(bpf_prog, insn); - if (!fm) - return -EINVAL; - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { u8 flags = fm->arg_flags[i]; u32 reg = BPF_REG_1 + i; @@ -2644,6 +2639,8 @@ st: insn_off = insn->off; /* call */ case BPF_JMP | BPF_CALL: { + const struct btf_func_model *fm = NULL; + func = (u8 *) __bpf_call_base + imm32; if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { LOAD_TAIL_CALL_CNT_PTR(stack_depth); @@ -2652,7 +2649,10 @@ st: insn_off = insn->off; if (!imm32) return -EINVAL; if (src_reg == BPF_PSEUDO_KFUNC_CALL) { - err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + err = emit_kfunc_arena_args(bpf_prog, fm, &prog); if (err < 0) return err; ip += err; @@ -2666,6 +2666,14 @@ st: insn_off = insn->off; return -EINVAL; if (priv_frame_ptr) pop_r9(&prog); + /* + * A kfunc returning more than 8 bytes hands the second + * half back in RDX (the native ABI's second return reg), + * but BPF expects it in R0:R2. BPF R0 is RAX (no move + * needed), while BPF R2 is RSI, so copy RDX into RSI. + */ + if (fm && fm->ret_size > 8) + emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_3); break; } @@ -4156,6 +4164,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/include/linux/filter.h b/include/linux/filter.h index 4a9bc6a848f2..6e746b0a0930 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1237,6 +1237,7 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d55e737ed75a..5db77d7915df 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3287,6 +3287,11 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return false; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2be5867d4412..48e31136a7df 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2916,6 +2916,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err; + if (func_model.ret_size > 8) { + if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { + verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", + kfunc.name); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_kfunc_ret_reg_pair()) { + verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", + kfunc.name); + return -EOPNOTSUPP; + } + } memset(&meta, 0, sizeof(meta)); meta.btf = kfunc.btf; From 5bbb242c0859024d6d0ba990c0476b33c7c0be71 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:59 -0700 Subject: [PATCH 14/90] bpf: Handle R2 as a return register in precision backtracking Precision backtracking treats only R0 as a return register at a call/return boundary, so once the verifier starts modeling R2 that way, marking the second half of such a return precise would trip the "unexpected regs" checks in backtrack_insn() and reject a valid program. Marking the upper half precise, for example by branching on it after a call to a static subprogram, walks backtracking into the callee and reaches its BPF_EXIT with R2 still set in the mask. Handle R2 like R0 in boundaries where a call defines the return registers. R2 differs from R0 in that it is an argument register as well, so it is part of the BPF_REGMASK_ARGS check and has to be cleared before that check rather than next to R0. Clear it unconditionally, rather than only where the callee or the kfunc really does return a pair. That gives up the "unexpected regs" assertion for R2, and in exchange keeps backtracking free of any BTF lookup. Nothing is lost: a callee that does not return a pair leaves the caller's R2 uninitialized, so the main verification pass has already rejected any program that reads it, and backtracking is never asked for its precision. At BPF_EXIT the return registers are sampled before the callback path clears R1-R5. That clear does not touch R0, but it does cover R2, and running it first would drop a pair return whenever the instruction following the call happens to be one that invokes a callback. Suggested-by: Eduard Zingerman Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055259.3295829-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/backtrack.c | 46 +++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index a2b18a9f1694..653db80bcc47 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -423,6 +423,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); + /* global subprog always sets R0 */ + bt_clear_reg(bt, BPF_REG_0); + /* and if it does not set R2, main pass would catch it */ + bt_clear_reg(bt, BPF_REG_2); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore @@ -432,8 +436,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - /* global subprog always sets R0 */ - bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which @@ -506,6 +508,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -ENOTSUPP; /* regular helper call sets R0 */ bt_clear_reg(bt, BPF_REG_0); + /* kfunc might also set R2 */ + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { /* if backtracking was looking for registers R1-R5 * they should have been found already. @@ -520,7 +524,25 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool r0_precise; + bool from_subprog_call, r0_precise, r2_precise; + + /* BPF_EXIT in subprog or callback always returns + * right after the call instruction, so by checking + * whether the instruction at subseq_idx-1 is subprog + * call or not we can distinguish actual exit from + * *subprog* from exit from *callback*. In the former + * case, we need to propagate the precision of the + * return registers, if necessary. In the latter we + * never do that. + */ + from_subprog_call = subseq_idx - 1 >= 0 && + bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + + /* Sample the return registers before the callback + * handling below clears R1-R5. + */ + r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); + r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. @@ -533,30 +555,22 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) for (i = BPF_REG_1; i <= BPF_REG_5; i++) bt_clear_reg(bt, i); + + bt_clear_reg(bt, BPF_REG_0); + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - /* BPF_EXIT in subprog or callback always returns - * right after the call instruction, so by checking - * whether the instruction at subseq_idx-1 is subprog - * call or not we can distinguish actual exit from - * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. - */ - r0_precise = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && - bt_is_reg_set(bt, BPF_REG_0); - - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; if (r0_precise) bt_set_reg(bt, BPF_REG_0); + if (r2_precise) + bt_set_reg(bt, BPF_REG_2); /* r6-r9 and stack slots will stay set in caller frame * bitmasks until we return back from callee(s) */ From 991fb7823106bb2faa8a4871bc1d7c745201bac9 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:04 -0700 Subject: [PATCH 15/90] bpf: Account R2 of register-pair returns in live register analysis A BPF_EXIT of a subprogram returning a value larger than 8 bytes (a struct/union or an __int128) reads R2 as well as R0, since the second half of the return value is passed back in R2. compute_insn_live_regs() only marked R0 used at exit, so a callee's R2 could be considered dead and cleaned from checkpointed states, which would allow unsound state pruning. Mark R2 as read at the BPF_EXIT of a subprogram that does return a register pair. bpf_compute_live_registers() now loops over the subprograms and, for each, over the [start, end) instruction range from env->subprog_info[], so the return convention is queried once per subprogram through bpf_ret_reg_pair() rather than once per instruction. Marking R2 at every exit instead would be simpler, but R2 would then stay live backwards across any call that is not followed by a write to R2, which is nearly every program, and would needlessly hurt state pruning. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055304.3296353-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/liveness.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 74fc4b3f80d6..71f998c6eb88 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -2060,7 +2060,8 @@ static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct insn_live_regs *info) + struct insn_live_regs *info, + bool ret_reg_pair) { struct bpf_call_summary cs; const u8 class = BPF_CLASS(insn->code); @@ -2072,6 +2073,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, const u32 src32 = mask_lo(src); const u32 dst32 = mask_lo(dst); const u32 r0 = reg64_mask(0); + const u32 r2 = reg64_mask(BPF_REG_2); u32 def = 0; u32 use = U32_MAX; @@ -2191,7 +2193,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_EXIT: def = 0; - use = r0; + use = ret_reg_pair ? (r0 | r2) : r0; break; case BPF_CALL: def = ALL_CALLER_SAVED_REGS; @@ -2228,8 +2230,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct insn_live_regs *state; int insn_cnt = env->prog->len; u64 pos, insn_pos; - int err = 0, i, j; - bool changed; + int err = 0, i, j, subprog, start, end; + bool changed, ret_reg_pair; /* Use the following algorithm: * - define the following: @@ -2256,8 +2258,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) goto out; } - for (i = 0; i < insn_cnt; ++i) - compute_insn_live_regs(env, &insns[i], &state[i]); + for (subprog = 0; subprog < env->subprog_cnt; subprog++) { + start = env->subprog_info[subprog].start; + end = env->subprog_info[subprog + 1].start; + ret_reg_pair = bpf_ret_reg_pair(env, subprog); + + for (i = start; i < end; ++i) + compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair); + } /* Forward pass: resolve stack access through FP-derived pointers */ err = bpf_compute_subprog_arg_access(env); From 0630ad00d96d9cf600c5c340cdecb363caa9d1cc Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:09 -0700 Subject: [PATCH 16/90] bpf: Add verifier support for 16-byte returns in R0: R2 LLVM 23 added support for returning a value in two registers for an __int128, or a struct/union whose size is greater than 8 but not more than 16 bytes. See LLVM patches [1] and [2]. Before LLVM 23 the BPF backend could not return these values at all. A by-value struct or union return (of any size) was rejected at compile time with: error: aggregate returns are not supported and an __int128 return failed later in the backend with: fatal error: error in backend: unable to allocate function return #1 Both are resolved in LLVM 23, which lowers such returns into the R0:R2 register pair. Model that pair at calls to global and static BPF subprograms and at kfunc calls: R2 is marked alongside R0 at the call, propagated out of a callee at its exit, and held to the same scalar-only and no-stack-pointer rules that already apply to R0. A struct returned by a kfunc must be composed of scalars, since its bytes reach the program as raw register contents and a pointer field would otherwise be laundered into a scalar. An extension program is the one caller of the convention that cannot take part in it: only R0 is checked at its exit, since that is the program exit code, so it has no way to hand back an upper half. Replacing a function whose return value is larger than 8 bytes is therefore rejected in btf_check_func_type_match(). That function compares btf_type->info, which carries no size for an int, so an 8 byte long and a 16 byte __int128 compared equal; sizes up to 8 bytes stay interchangeable, as they always have been, so this only rejects what the R0:R2 convention newly makes incompatible. [1] https://github.com/llvm/llvm-project/pull/190894 [2] https://github.com/llvm/llvm-project/pull/206876 Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055309.3297684-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/btf.c | 6 ++++ kernel/bpf/verifier.c | 68 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index da36d4b9d31a..a10a070016be 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7677,6 +7677,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, btf_type_str(t2), fn2); return -EINVAL; } + if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) { + bpf_log(log, + "Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n", + fn1, t1->size, fn2, t2->size); + return -EINVAL; + } for (i = 0; i < nargs1; i++) { t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 48e31136a7df..930925fada30 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -415,6 +415,9 @@ static u32 ret_regs_cnt(u32 size) return size > 8 && size <= 16 ? 2 : 1; } +/* Registers holding a function return value, in order. See ret_regs_cnt(). */ +static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 }; + static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) { const struct btf *btf = env->prog->aux->btf; @@ -9930,6 +9933,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; int err, subprog, target_insn; + u32 i, nregs; target_insn = *insn_idx + insn->imm + 1; subprog = bpf_find_subprog(env, target_insn); @@ -9985,9 +9989,14 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); - /* All non-void global functions return a 64-bit SCALAR_VALUE. */ + /* + * A non-void global function returns a 64-bit SCALAR_VALUE in + * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair. + */ if (!returns_void) { - mark_reg_unknown(env, caller->regs, BPF_REG_0); + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) + mark_reg_unknown(env, caller->regs, ret_regs[i]); bpf_diag_mod_end(env); } @@ -10347,11 +10356,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) struct bpf_func_state *caller, *callee; struct bpf_reg_state *r0; bool in_callback_fn; + u32 i, nregs; int err; callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; - if (r0->type == PTR_TO_STACK) { + nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1; + for (i = 0; i < nregs; i++) { + if (callee->regs[ret_regs[i]].type != PTR_TO_STACK) + continue; /* technically it's ok to return caller's stack pointer * (or caller's caller's pointer) back to the caller, * since these pointers are valid. Only current stack @@ -10386,9 +10399,13 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return -EFAULT; } } else { - /* return to the caller whatever r0 had in the callee */ + /* + * return to the caller whatever the callee had in the + * return register(s) + */ bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - caller->regs[BPF_REG_0] = *r0; + for (i = 0; i < nregs; i++) + caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; bpf_diag_mod_end(env); } @@ -11323,6 +11340,19 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } +/* + * Mark the register(s) holding a @size byte kfunc return value as unknown + * scalars. Both halves of a register pair are treated the same way. + */ +static void mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) +{ + u32 i, nregs = ret_regs_cnt(size); + + for (i = 0; i < nregs; i++) + mark_reg_unknown(env, regs, ret_regs[i]); +} + static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; @@ -13991,10 +14021,25 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_reg_unknown(env, regs, BPF_REG_0); + mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); + } else if (btf_type_is_struct(t)) { + /* + * The returned struct comes back as raw register bits modeled + * as an unknown scalar, so it must contain only scalars: + * otherwise a pointer field would be laundered into a scalar + * and escape provenance and reference tracking. + */ + if (!__btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + verbose(env, + "kernel function %s returns %s %s that is not composed of scalars\n", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off)); + return -EINVAL; + } + mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -17562,11 +17607,20 @@ static int check_global_subprog_return_code(struct bpf_verifier_env *env) { struct bpf_func_state *cur_frame = cur_func(env); u32 subprog = cur_frame->subprogno; + u32 i, nregs; + int err; if (subprog_returns_void(env, subprog)) return 0; - return check_global_ret_scalar_reg(env, BPF_REG_0); + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) { + err = check_global_ret_scalar_reg(env, ret_regs[i]); + if (err) + return err; + } + + return 0; } /* Bitmask with 1s for all caller saved registers */ From a2e98481a6396f919ad91ba9c2ff3ec61c4028b3 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:14 -0700 Subject: [PATCH 17/90] bpf: Enable aggregate return types up to 16 bytes Relax btf_distill_func_proto() to accept a by-value struct or union that the R0:R2 convention added in earlier patches can carry: - a struct or union larger than 8 and up to 16 bytes, returned in the R0:R2 register pair, matching what LLVM emits for the BPF target; - a struct or union up to 8 bytes, returned in R0 alone. A >8 byte scalar (__int128) was already accepted and is unchanged. Everything else stays rejected: a return type larger than 16 bytes, and any type that __get_type_size() cannot return in registers at all (e.g. an array), which it already reports as ret < 0. btf_validate_return_type() is relaxed as well, so that it accepts a by-value struct or union up to 16 bytes in addition to void and scalars. With btf_distill_func_proto() and btf_validate_return_type() relaxed, the verifier, JIT, precision-backtracking and live-register support from the earlier patches becomes reachable: <=16 byte aggregate return values now work end to end. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055314.3298664-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/btf.c | 23 +++++++++++++++---- kernel/bpf/verifier.c | 18 +++++++-------- .../selftests/bpf/progs/exceptions_fail.c | 2 +- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 953d8191563a..004b06785521 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,6 +1488,8 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a10a070016be..91b8ce77f699 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7584,7 +7584,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0 || btf_type_is_struct(t)) { + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s return type %s is unsupported.\n", tname, btf_type_str(t)); @@ -7963,7 +7963,7 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, - const struct btf_type *t, int subprog) + const struct btf_type *t, int subprog, bool is_global) { u32 tags = 0; int err; @@ -7986,6 +7986,19 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) return 0; + if (btf_type_is_struct(t) && t->size <= 16) { + /* + * A global function's caller models the return as an opaque + * scalar pair, so it may only return scalars by value. A local + * function is verified inline, so a pointer field stays tracked + * and needs no such restriction. + */ + bool local_func = subprog && !is_global; + + if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + return 0; + } + return -EOPNOTSUPP; } @@ -8073,12 +8086,12 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - err = btf_validate_return_type(env, btf, t, subprog); + err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", + "Global function %s() has unsupported return type. " + "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", tname); } return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 930925fada30..e036ae20bf6b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11620,9 +11620,9 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11640,7 +11640,7 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) return false; continue; } @@ -12039,7 +12039,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12055,7 +12055,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13111,7 +13111,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; } - if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13653,7 +13653,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14032,7 +14032,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!__btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index ac44d60e5066..9708efb93683 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Global function exception_cb_bad_ret_type1() return value not void or scalar.") +__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); From 7b7b511e0cf53d2602ba6119c9a52174d548ff26 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:20 -0700 Subject: [PATCH 18/90] selftests/bpf: Add C tests for 16-byte returns in R0: R2 Add selftests that exercise a 16-byte return value passed in the R0:R2 register pair, written in C so that they depend on the compiler lowering the register-pair return. The R0:R2 convention is only emitted by LLVM 23 and newer, and a by-value aggregate return does not compile at all before that, so the programs sit behind a __clang_major__ guard. An older compiler builds the dummy test in the #else branch instead, which keeps the object non-empty and says in its description why nothing was exercised. The kfunc tests are tagged __arch_x86_64/__arch_arm64 and skip elsewhere. Those are the architectures whose JIT advertises bpf_jit_supports_kfunc_ret_reg_pair(), which bpf_add_kfunc_call() requires before it accepts a kfunc returning more than 8 bytes, and they are also the only ones building the kfuncs. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055320.3300114-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- .../selftests/bpf/prog_tests/verifier.c | 2 + .../bpf/progs/verifier_aggregate_ret.c | 178 ++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 18 ++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 9 + 4 files changed, 207 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 64ac49ad67e6..f7f94ccebce2 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -5,6 +5,7 @@ #include "arena_kfunc.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "verifier_aggregate_ret.skel.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" #include "verifier_arena.skel.h" @@ -170,6 +171,7 @@ void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } +void test_verifier_aggregate_ret(void) { RUN_TESTS(verifier_aggregate_ret); } void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c new file mode 100644 index 000000000000..7851bade2b40 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +typedef unsigned __int128 u128; + +struct pair { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ + } parts; +}; + +static __noinline u128 make_i128(__u64 a, __u64 b) +{ + return ((u128)(a + b) << 64) | (a - b); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = make_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +static __noinline struct pair make_pair(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +__noinline struct pair make_pair_global(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair_global(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +static __noinline union upair make_upair(__u64 a, __u64 b) +{ + union upair p; + + p.halves[0] = a + b; + p.halves[1] = a - b; + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair p; + + p = make_upair(a, b); + if (p.parts.lo != a + b) + return 1; + if (p.parts.hi != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = bpf_kfunc_call_test_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_ret_pair p; + + p = bpf_kfunc_call_test_ret_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +#else + +SEC("socket") +__description("verifier_aggregate_ret: needs LLVM 23, dummy test") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 9366a3c578f1..c4bc8e11c6e1 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -954,6 +954,20 @@ __bpf_kfunc int bpf_kfunc_call_test5(u8 a, u16 b, u32 c) return 0; } +#if defined(__x86_64__) || defined(__aarch64__) +__bpf_kfunc __int128 bpf_kfunc_call_test_i128(u64 a, u64 b) +{ + return (__int128)(((unsigned __int128)(a + b) << 64) | (a - b)); +} + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} +#endif /* __x86_64__ || __aarch64__ */ + __bpf_kfunc u64 bpf_kfunc_call_stack_arg(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, u64 h, u64 i, u64 j) @@ -1487,6 +1501,10 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3) BTF_ID_FLAGS(func, bpf_kfunc_call_test4) BTF_ID_FLAGS(func, bpf_kfunc_call_test5) +#if defined(__x86_64__) || defined(__aarch64__) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) +#endif BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_ptr) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mix) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 7d81070eefe7..426000f9a14f 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -55,6 +55,11 @@ struct prog_test_big_arg { __u64 b; }; +struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ + __u64 lo; + __u64 hi; +}; + struct prog_test_fail1 { void *p; int x; @@ -131,6 +136,10 @@ int bpf_kfunc_call_test2(struct sock *sk, __u32 a, __u32 b) __ksym; struct sock *bpf_kfunc_call_test3(struct sock *sk) __ksym; long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; int bpf_kfunc_call_test5(__u8 a, __u16 b, __u32 c) __ksym; +#ifdef __SIZEOF_INT128__ +__int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; +#endif +struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, __u64 i, __u64 j) __ksym; From 0b9fc795f83f5831de858d659de1097800ba7f18 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:25 -0700 Subject: [PATCH 19/90] selftests/bpf: Add inline-asm and subprog tests for R0: R2 returns Add inline-asm tests covering what the C tests cannot reach, since they need a callee that violates the convention on purpose. The coverage includes BPF-to-BPF returns, kfunc calls, backtracking and liveness. In addition, a negative freplace test checks that an extension cannot replace a function returning R0:R2. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055325.3300712-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- .../selftests/bpf/prog_tests/aggregate_ret.c | 11 + .../selftests/bpf/prog_tests/fexit_bpf2bpf.c | 16 ++ .../selftests/bpf/progs/aggregate_ret_func.c | 237 ++++++++++++++++++ .../selftests/bpf/progs/aggregate_ret_kfunc.c | 121 +++++++++ .../bpf/progs/aggregate_ret_target.c | 29 +++ .../bpf/progs/compute_live_registers.c | 28 +++ .../selftests/bpf/progs/freplace_ret_pair.c | 12 + .../selftests/bpf/test_kmods/bpf_testmod.c | 37 +++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 20 ++ 9 files changed, 511 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/aggregate_ret.c create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_func.c create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_target.c create mode 100644 tools/testing/selftests/bpf/progs/freplace_ret_pair.c diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c new file mode 100644 index 000000000000..e0b94ed10f94 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include "aggregate_ret_func.skel.h" +#include "aggregate_ret_kfunc.skel.h" + +void test_aggregate_ret(void) +{ + RUN_TESTS(aggregate_ret_func); + RUN_TESTS(aggregate_ret_kfunc); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 2523c07a16c6..6262c81cdaa9 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -441,6 +441,20 @@ static void test_func_replace_int_with_void(void) " doesn't match type INT of global_func2()"); } +static void test_func_replace_ret_pair(void) +{ + const char *msg = "Return type of new_agg_ret_target_func() has size 8 " + "and of agg_ret_target_func() size 16"; + + /* + * An extension cannot replace a function whose return value comes back + * in the R0:R2 pair: only R0 is checked at the extension's exit, so it + * would leave R2 stale for the target's callers. + */ + test_obj_load_failure_common("freplace_ret_pair.bpf.o", + "./aggregate_ret_target.bpf.o", msg); +} + static int find_prog_btf_id(const char *name, __u32 attach_prog_fd) { struct bpf_prog_info info = {}; @@ -660,6 +674,8 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace_progmap(); if (test__start_subtest("freplace_int_with_void")) test_func_replace_int_with_void(); + if (test__start_subtest("freplace_ret_pair")) + test_func_replace_ret_pair(); if (test__start_subtest("freplace_void")) test_func_replace_void(); if (test__start_subtest("sleepable_fentry_to_xdp")) diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c new file mode 100644 index 000000000000..6f66fc822ced --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +typedef unsigned __int128 u128; + +__naked u128 global_agg_good(void) +{ + asm volatile ( + "r0 = 0x1234;" /* low 64 bits */ + "r2 = 0x5678;" /* high 64 bits */ + "exit;" + ); +} + +__naked u128 global_agg_bad(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked u128 global_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_global_fail(void) +{ + asm volatile ( + "call %[global_agg_bad];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__failure __msg("At subprogram exit the register R2 is not a scalar value") +__naked int aggregate_ret_global_ptr_fail(void) +{ + asm volatile ( + "call %[global_agg_bad_ptr];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" /* stack pointer placed in the second return register */ + "exit;" + ); +} + +/* + * R2 is a return register once the subprogram returns a pair, so a stack + * pointer left in it is rejected at the callee's exit exactly as one in R0 + * is: the callee frame is gone by the time the caller could use it. + */ +SEC("tc") +__load_if_JITed() +__failure __msg("cannot return stack pointer to the caller") +__naked int aggregate_ret_static_ptr_fail(void) +{ + asm volatile ( + "call %[static_agg_bad_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_no_r2(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_static_uninit_fail(void) +{ + asm volatile ( + "call %[static_agg_no_r2];" + "r0 = r2;" + "exit;" + : + : __imm(static_agg_no_r2) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_precise(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 4;" /* second half; its value is made precise below */ + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r2 stack= before 11: (b7) r2 = 4") +__naked int aggregate_ret_static_precise(void) +{ + asm volatile ( + "call %[static_agg_precise];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_precise) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 0: (85) call pc+9") +__naked int aggregate_ret_global_precise(void) +{ + asm volatile ( + "call %[global_agg_good];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(global_agg_good) + : __clobber_all); +} + +#if defined(__clang_major__) && __clang_major__ >= 23 + +/* A by-value struct that smuggles a pointer, which must be rejected. */ +struct with_ptr { + void *p; + __u64 x; +}; + +/* A by-value union that smuggles a pointer, which must be rejected too. */ +union upair_with_ptr { + void *p; + __u64 halves[2]; +}; + +__naked struct with_ptr global_ret_struct_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_struct_ptr() has unsupported return type") +__naked int aggregate_ret_global_struct_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_struct_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_struct_ptr) + : __clobber_all); +} + +__naked union upair_with_ptr global_ret_union_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_union_ptr() has unsupported return type") +__naked int aggregate_ret_global_union_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_union_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_union_ptr) + : __clobber_all); +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c new file mode 100644 index 000000000000..d6b422ae9784 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Reference kfunc addresses to force those BTF to be emitted. Taking the address + * (rather than calling) avoids any dependence on the compiler lowering an + * __int128 or struct return value, which the BPF backend only supports from + * LLVM 23 on. + */ +void __kfunc_btf_root(void) +{ + asm volatile ("" + : + : "r"(&bpf_kfunc_call_test_i128), + "r"(&bpf_kfunc_call_test_ret_fastcall), + "r"(&bpf_kfunc_call_test_ret_ptr), + "r"(&bpf_kfunc_call_test_ret_ii), + "r"(&bpf_kfunc_call_test_ret_big)); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 7 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 6: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 5: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 2: (85) call bpf_kfunc_call_test_i128") +__naked int aggregate_ret_kfunc_precise(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_i128];" + "r6 = r2;" /* second return half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_i128) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("kfunc bpf_kfunc_call_test_ret_fastcall with >8-byte return is not supported with KF_FASTCALL") +__naked int aggregate_ret_kfunc_fastcall_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_ret_fastcall];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_fastcall) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars") +__naked int aggregate_ret_kfunc_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_kfunc_small_no_r2(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "call %[bpf_kfunc_call_test_ret_ii];" + "r0 = r2;" /* R2 is not a return register for a <=8 byte struct */ + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ii) + : __clobber_all); +} + +/* + * A return value larger than 16 bytes does not fit in R0:R2 and is rejected by + * btf_distill_func_proto() before the KF_FASTCALL and JIT-capability checks. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("The function bpf_kfunc_call_test_ret_big return type STRUCT is unsupported") +__naked int aggregate_ret_kfunc_too_big_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_big];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_big) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_target.c b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c new file mode 100644 index 000000000000..cffd8d7d3241 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +/* freplace target: a global subprogram returning 16 bytes in R0:R2. */ +__naked unsigned __int128 agg_ret_target_func(void) +{ + asm volatile ( + "r0 = 0x1234;" + "r2 = 0x5678;" + "exit;" + ); +} + +SEC("tc") +__naked int agg_ret_target(void) +{ + asm volatile ( + "call %[agg_ret_target_func];" + "r0 = 0;" + "exit;" + : + : __imm(agg_ret_target_func) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index d055fc7b3b95..338e67cd8856 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -431,6 +431,34 @@ __naked void subprog1(void) ::: __clobber_all); } +static __used __naked unsigned __int128 aux2(void) +{ + asm volatile ( + "r0 = 1;" + "r2 = 2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +/* A program observing the pair needs the JIT; see bpf_compute_subprog_ret_regs(). */ +__load_if_JITed() +__log_level(2) +__msg("0: .12345.... (85) call pc+2") +__msg("1: ..2....... (bf) r0 = r2") +__msg("2: 0......... (95) exit") +__msg("3: .......... (b7) r0 = 1") +__msg("4: 0......... (b7) r2 = 2") +__msg("5: 0.2....... (95) exit") +__naked void subprog_ret_reg_pair(void) +{ + asm volatile ( + "call aux2;" + "r0 = r2;" + "exit;" + ::: __clobber_all); +} + #if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/freplace_ret_pair.c b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c new file mode 100644 index 000000000000..12c15d293bd7 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +SEC("freplace/agg_ret_target_func") +__u64 new_agg_ret_target_func(void) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index c4bc8e11c6e1..850cf4f830c4 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -966,8 +966,41 @@ __bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(u64 a, u64 b) return r; } + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) +{ + struct prog_test_ret_ptr r = { .p = NULL, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) +{ + struct prog_test_ret_ii r = { .a = a, .b = b }; + + return r; +} #endif /* __x86_64__ || __aarch64__ */ +/* + * Takes no argument on purpose: with no arguments there is nothing for the sret + * pointer to displace, so this needs no architecture guard even though it + * returns 24 bytes. + */ +__bpf_kfunc struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) +{ + struct prog_test_ret_big r = { .a = 1, .b = 2, .c = 3 }; + + return r; +} + __bpf_kfunc u64 bpf_kfunc_call_stack_arg(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, u64 h, u64 i, u64 j) @@ -1504,7 +1537,11 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test5) #if defined(__x86_64__) || defined(__aarch64__) BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) #endif +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_ptr) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mix) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 426000f9a14f..65e693ada736 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -60,6 +60,22 @@ struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ __u64 hi; }; +struct prog_test_ret_ii { /* 8 bytes: R0 only */ + int a; + int b; +}; + +struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ + void *p; + __u64 tag; +}; + +struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ + __u64 a; + __u64 b; + __u64 c; +}; + struct prog_test_fail1 { void *p; int x; @@ -140,6 +156,10 @@ int bpf_kfunc_call_test5(__u8 a, __u16 b, __u32 c) __ksym; __int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; #endif struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; +struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; +struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; +struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, __u64 i, __u64 j) __ksym; From d50d9a06c86a76e800071d184d05c9cdbfa1ec4b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:30 -0700 Subject: [PATCH 20/90] Documentation/bpf: Document up to 16-byte kfunc return values in R0: R2 kfuncs may now return a value larger than 8 bytes and up to 16 bytes (a scalar-only struct or union, or an __int128), passed back in the R0:R2 register pair. Add a kfunc return-value section documenting this, including that a struct or union up to 8 bytes is returned in R0 alone, which struct and union members are accepted, that the R0:R2 register pair requires JIT support (bpf_jit_supports_kfunc_ret_reg_pair()), and that a return value larger than 16 bytes is unsupported. Also note that the same convention applies to BPF subprogram returns, and document the consequence for a global subprogram: it must assign both halves of a register-pair return, since an unassigned R2 may be left holding a pointer argument and is then rejected as a leak. Signed-off-by: Yonghong Song Link: https://patch.msgid.link/20260819055330.3301120-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- Documentation/bpf/kfuncs.rst | 65 ++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 85f73e0bbd0f..89dea6b0b024 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -575,6 +575,71 @@ is also covered by this recovery. A kfunc handed an arena pointer may therefore access up to ``GUARD_SZ / 2`` past it without bounds-checking against the arena. Larger accesses must verify the range explicitly. +2.9 kfunc Return Values +----------------------- + +A kfunc may return a scalar, a pointer, or a small struct or union by +value. A scalar or pointer of up to 8 bytes is returned in R0, as usual. + +A struct or union returned by value must be composed only of scalars +(recursively), where a scalar is an integer or an enum; arrays of scalars are +allowed as members. Its bytes are handed back to the program as the raw +contents of R0 (and R2), so a pointer field would be laundered into a scalar +and escape the verifier's pointer provenance and reference tracking. A struct +or union with a pointer member is therefore rejected at load time, and so is +one with a floating-point member, which the ABI may not return in R0:R2 at +all. + +A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a +scalar-only struct or union, or an ``__int128``. Such a value is returned +in the register pair R0:R2, matching the convention LLVM uses for the BPF +target: the first 8 bytes in R0 and the second 8 bytes in R2. A struct or +union of 8 bytes or less is returned in R0 alone. + +:: + + struct bpf_pair { __u64 a, b; }; /* 16 bytes */ + + __bpf_kfunc struct bpf_pair bpf_kfunc_get_pair(void) + { + struct bpf_pair p = { .a = 1, .b = 2 }; + + return p; /* p.a in R0, p.b in R2 */ + } + +Returning a value in the R0:R2 pair requires the JIT to place the second +half of the return value into R2, which not every architecture supports +right now. A kfunc with a return value larger than 8 bytes is therefore +rejected at load time on a JIT that does not advertise this capability (see +``bpf_jit_supports_kfunc_ret_reg_pair()``), and such a program is never run +by the interpreter. A return value larger than 16 bytes is not supported. + +The same R0:R2 convention applies to a BPF subprogram, global or static, that +returns an ``__int128`` or a struct or union larger than 8 bytes. It is only +used when the program is JITed, since the interpreter propagates only R0 out of +a subprogram: without a JIT the return value stays in R0 alone, and a caller +reading R2 is rejected for reading an uninitialized register. A global +subprogram is verified in isolation, so its by-value struct or union return is +restricted to scalars just like a kfunc's; a static subprogram is verified +inline and has no such restriction. The main program is not covered: its return +value is the program's exit code, read out of R0 alone, so a declared upper +half is never looked at. + +A global subprogram must leave a scalar in *every* register of the pair, so +both halves of the returned value have to be assigned. Leaving the upper half +uninitialized is not merely untidy: the compiler is then free to leave R2 +holding whatever it happened to hold, which for a subprogram taking a pointer +argument is typically that pointer. Handing the caller an unknown scalar built +from a pointer is a leak, so the verifier rejects it with:: + + At subprogram exit the register R2 is not a scalar value (...) + +Initialize the whole return value, for example ``struct pair p = {};``, to +avoid this. A static subprogram is exempt from the scalar-only rule: it is +verified inline, so an unassigned R2 is simply passed back to the caller as +uninitialized and only a caller that reads it fails. A stack pointer left in +R2 is still rejected there, just as one in R0 is. + .. _BPF_kfunc_lifecycle_expectations: 3. kfunc lifecycle expectations From e40f61a31688898e7cf81e91074a42f805883e4d Mon Sep 17 00:00:00 2001 From: Darren Carreras Date: Thu, 13 Aug 2026 20:43:18 -0400 Subject: [PATCH 21/90] libbpf: Avoid overflow in BTF.ext bounds check A malformed BTF.ext subsection length in an ELF input can wrap the pointer addition used by btf_ext_parse_sec_info() on 32-bit builds. The wrapped pointer passes the bounds check and parsing then reads beyond the copied BTF.ext data. Ensure the header fits in the copied data, then validate the subsection offset and length with subtraction before forming its pointer. Fixes: ae4ab4b4117d ("btf: expose API to work with raw btf_ext data") Closes: https://issues.oss-fuzz.com/issues/477315119 Signed-off-by: Darren Carreras Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260813-b4-libbpf-v6-send-20260813-v6-1-56be61c52758@gmail.com --- tools/lib/bpf/btf.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 8417de92d028..c783359977b4 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -3364,7 +3364,7 @@ static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, { const struct btf_ext_info_sec *sinfo; struct btf_ext_info *ext_info; - __u32 info_left, record_size; + __u32 data_left, info_left, record_size; size_t sec_cnt = 0; void *info; @@ -3377,16 +3377,20 @@ static int btf_ext_parse_sec_info(struct btf_ext *btf_ext, return -EINVAL; } - /* The start of the info sec (including the __u32 record_size). */ - info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; - info_left = ext_sec->len; + if (btf_ext->hdr->hdr_len > btf_ext->data_size) + return -EINVAL; - if (btf_ext->data + btf_ext->data_size < info + ext_sec->len) { + data_left = btf_ext->data_size - btf_ext->hdr->hdr_len; + if (ext_sec->off > data_left || ext_sec->len > data_left - ext_sec->off) { pr_debug("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", ext_sec->desc, ext_sec->off, ext_sec->len); return -EINVAL; } + /* The start of the info sec (including the __u32 record_size). */ + info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off; + info_left = ext_sec->len; + /* At least a record size */ if (info_left < sizeof(__u32)) { pr_debug(".BTF.ext %s record size not found\n", ext_sec->desc); From 669e4fa766000ae4137bb02eb22ce75ba78ad32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Tue, 18 Aug 2026 20:43:00 +0200 Subject: [PATCH 22/90] selftests/bpf: Allocate a larger timeout for connection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some tests, like tc_tunnel or tc_edt, sporadically fail in CI with the following logs: (network_helpers.c:309: errno: Operation now in progress) \ Failed to connect to server send_and_test_data:FAIL:connect to server unexpected error: -115 This is due to SO_RCVTIMEO and SO_SNDTIMEO being set on the client socket (see settimeo() in client_socket()), allowing connect() to return an error and to set errno to EINPROGRESS instead of ETIMEDOUT. Increasing the timeout value for those tests is likely not a good solution (and it has already been done by commit 2790db208b44 ("selftests/bpf: Improve tc_tunnel test reliability")): some tests expect some data transfer to fail, and so the timeout value would increase overall test execution duration again (not only the connection, but any socket operation). Another solution is to allocate a timeout budget specific to the connection: we can apply a larger timeout only for connections, and once the connection is established, set back the timeout configured through opts->timeout_ms; this would allow connection to succeed under heavy CI load, while keeping timeout reasonable for the rest of the test traffic. Create a dedicated connect helper that sets a large timeout for the connect step, tries to connect, and if successful, finally sets the timeout value to the one configured by the test. Fixes: 99126abec5e5 ("bpf: selftests: A few improvements to network_helpers.c") Signed-off-by: Alexis Lothoré (eBPF Foundation) Signed-off-by: Andrii Nakryiko Acked-by: Ihor Solodrai Link: https://lore.kernel.org/bpf/20260818-tc_tunnel_flaky-v7-1-aec161e74ef2@bootlin.com --- tools/testing/selftests/bpf/network_helpers.c | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index cdf2d7d3ab32..d27dc79475de 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -49,6 +49,8 @@ errno = __save; \ }) +#define CONNECT_MIN_TIMEOUT_MS 5000 + struct ipv4_packet pkt_v4 = { .eth.h_proto = __bpf_constant_htons(ETH_P_IP), .iph.ihl = 5, @@ -291,6 +293,37 @@ int client_socket(int family, int type, return -1; } +static int connect_with_timeout(int fd, const struct sockaddr_storage *addr, + socklen_t addrlen, int timeout_ms) +{ + int connect_timeout_ms = MAX(timeout_ms, CONNECT_MIN_TIMEOUT_MS); + + /* + * Override timeout configuration with a larger value for the + * connection + */ + if (settimeo(fd, connect_timeout_ms)) { + log_err("Failed to set connect timeout"); + return -1; + } + + if (connect(fd, (const struct sockaddr *)addr, addrlen)) { + log_err("Failed to connect"); + return -1; + } + + /* + * If the timeout configured by the test is different from the + * connect timeout, restore it + */ + if (timeout_ms != connect_timeout_ms && settimeo(fd, timeout_ms)) { + log_err("Failed to set timeout for connected socket"); + return -1; + } + + return 0; +} + int connect_to_addr(int type, const struct sockaddr_storage *addr, socklen_t addrlen, const struct network_helper_opts *opts) { @@ -305,8 +338,7 @@ int connect_to_addr(int type, const struct sockaddr_storage *addr, socklen_t add return -1; } - if (connect(fd, (const struct sockaddr *)addr, addrlen)) { - log_err("Failed to connect to server"); + if (connect_with_timeout(fd, addr, addrlen, opts->timeout_ms)) { save_errno_close(fd); return -1; } @@ -376,20 +408,12 @@ int connect_fd_to_fd(int client_fd, int server_fd, int timeout_ms) struct sockaddr_storage addr; socklen_t len = sizeof(addr); - if (settimeo(client_fd, timeout_ms)) - return -1; - if (getsockname(server_fd, (struct sockaddr *)&addr, &len)) { log_err("Failed to get server addr"); return -1; } - if (connect(client_fd, (const struct sockaddr *)&addr, len)) { - log_err("Failed to connect to server"); - return -1; - } - - return 0; + return connect_with_timeout(client_fd, &addr, len, timeout_ms); } int make_sockaddr(int family, const char *addr_str, __u16 port, From bd2466ed8fc2ee54408bfe433e75f9f46bfa04f7 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:24 +0800 Subject: [PATCH 23/90] bpf: Roll back stream capacity when allocation fails bpf_stream_push_str() accounts the string length before allocating a stream element. If the allocation fails, the length remains charged even though no element is queued and therefore cannot be released by a reader. Repeated failures can exhaust the stream capacity permanently until the BPF program is freed. Refactor bpf_stream_release_capacity() to take a length so the consume and release sides are symmetric, and use it to roll back the charge when creating the stream element fails. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/c0ad7b18c59be6695fcc9e4e1200b2f4d5cbbea2.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index be9ce98e9469..0b157ec4e38e 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -68,10 +68,8 @@ static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len) return 0; } -static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem) +static void bpf_stream_release_capacity(struct bpf_stream *stream, int len) { - int len = elem->total_len; - atomic_sub(len, &stream->capacity); } @@ -79,7 +77,14 @@ static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int l { int ret = bpf_stream_consume_capacity(stream, len); - return ret ?: __bpf_stream_push_str(&stream->log, str, len); + if (ret) + return ret; + + ret = __bpf_stream_push_str(&stream->log, str, len); + if (ret) + bpf_stream_release_capacity(stream, len); + + return ret; } static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux) @@ -188,7 +193,7 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) if (cont) continue; bpf_stream_backlog_pop(stream); - bpf_stream_release_capacity(stream, elem); + bpf_stream_release_capacity(stream, elem->total_len); bpf_stream_free_elem(elem); } From effa6370ba89e7839c9bfeb802d8f05d6843ddf1 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:25 +0800 Subject: [PATCH 24/90] bpf: Fix stream capacity leak in staging path bpf_stream_stage_printk() increments ss->len before pushing the formatted string to the staging log. If element allocation fails, ss->len remains inflated and bpf_stream_stage_commit() permanently charges the stream capacity for data that was never queued. Only account the string length after a successful push, and use vscnprintf() so the staged length is the truncated payload without the trailing NUL. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/46898fb1ae482b2f8b1da459922b187378dfdc1d.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 0b157ec4e38e..a36fc9338967 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -316,17 +316,18 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) { struct bpf_bprintf_buffers *buf; va_list args; - int ret; + int len, ret; if (bpf_try_get_buffers(&buf)) return -EBUSY; va_start(args, fmt); - ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); + len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); va_end(args); - ss->len += ret; /* Exclude NULL byte during push. */ - ret = __bpf_stream_push_str(&ss->log, buf->buf, ret); + ret = __bpf_stream_push_str(&ss->log, buf->buf, len); + if (!ret) + ss->len += len; bpf_put_buffers(); return ret; } From 16790357a50baae10de1f619b608a85145d54bc2 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:26 +0800 Subject: [PATCH 25/90] bpf: Return partial progress from bpf_stream_read on fault bpf_stream_read() pops and frees stream elements after a successful copy_to_user(). If a later copy_to_user() fails, it currently restores only the current element's consumed_len and returns -EFAULT, hiding bytes already delivered to userspace and making the consumed data unrecoverable on retry. On a short copy, keep the successfully copied prefix of the current element and return the number of bytes copied. Return -EFAULT only when no bytes were copied for the call. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/7668c5838c6beafeea719df8d55319e2fe08cfef.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index a36fc9338967..09471a605268 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -167,6 +167,7 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) while (rem_len) { int pos = len - rem_len; + int chunk, n; bool cont; node = bpf_stream_backlog_peek(stream); @@ -180,13 +181,14 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) cons_len = elem->consumed_len; cont = bpf_stream_consume_elem(elem, &rem_len) == false; + chunk = elem->consumed_len - cons_len; - ret = copy_to_user(buf + pos, elem->str + cons_len, - elem->consumed_len - cons_len); - /* Restore in case of error. */ - if (ret) { - ret = -EFAULT; - elem->consumed_len = cons_len; + n = copy_to_user(buf + pos, elem->str + cons_len, chunk); + if (n) { + /* Keep any successfully copied bytes; -EFAULT only if none. */ + elem->consumed_len -= n; + rem_len += n; + ret = (len == rem_len) ? -EFAULT : 0; break; } From 41c0348f6e65c26df4866d5dc96c57b1995fe36c Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:27 +0800 Subject: [PATCH 26/90] bpf: Reject oversized bpf_stream_vprintk output with -E2BIG bstr_printf() returns the would-be length excluding the trailing NUL. When that value is >= MAX_BPRINTF_BUF the message was truncated, but bpf_stream_push_str() still tried to allocate with the inflated length and failed with -ENOMEM. The boundary case of exactly MAX_BPRINTF_BUF could also copy the trailing NUL into the stream element. Reject such lengths with -E2BIG before charging stream capacity, and tighten bpf_stream_elem_alloc() to accept only payloads strictly shorter than the bprintf buffer. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/74387ae5024d7e2996095e97211430285095b525.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 09471a605268..7a5c3ac8676b 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -22,11 +22,11 @@ static struct bpf_stream_elem *bpf_stream_elem_alloc(int len) size_t alloc_size; /* - * Length denotes the amount of data to be written as part of stream element, - * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can - * accomodate, therefore deny allocations that won't fit into them. + * Length is the payload pushed into the stream, excluding the + * trailing NUL of the bprintf buffer. Reject anything that cannot + * fit without copying that NUL into the stream element. */ - if (len < 0 || len > max_len) + if (len < 0 || len >= max_len) return NULL; alloc_size = offsetof(struct bpf_stream_elem, str[len]); @@ -245,6 +245,11 @@ __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const vo return ret; ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args); + /* Truncation: reject before capacity charge (not -ENOMEM). */ + if (ret >= MAX_BPRINTF_BUF) { + bpf_bprintf_cleanup(&data); + return -E2BIG; + } /* Exclude NULL byte during push. */ ret = bpf_stream_push_str(stream, data.buf, ret); bpf_bprintf_cleanup(&data); From 0cf194ccf78852377458b10c8c2e308678b10efa Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:28 +0800 Subject: [PATCH 27/90] selftests/bpf: Cover stream capacity and partial read edge cases Add coverage for the stream fixes requested on the capacity rollback series: - oversized bpf_stream_printk() returns -E2BIG and does not leak capacity for a subsequent successful write on the same program; - bpf_prog_stream_read() returns the successfully copied prefix when the userspace buffer straddles an unmapped page. Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/c5a79efbd3cb52cc0668ab50db5379c907cfe6a6.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../testing/selftests/bpf/prog_tests/stream.c | 63 +++++++++++++++++++ tools/testing/selftests/bpf/progs/stream.c | 28 +++++++++ 2 files changed, 91 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index c3cce5c292bd..e4caa60b4f9b 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -58,6 +58,69 @@ void test_stream_syscall(void) stream__destroy(skel); } +void test_stream_oversize(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct stream *skel; + int ret, prog_fd; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_oversize); + ret = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_OK(ret, "oversize run"); + ASSERT_OK(opts.retval, "oversize retval"); + + stream__destroy(skel); +} + +void test_stream_partial_read(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct stream *skel; + int ret, prog_fd; + long page_size; + char *page, *buf; + char rest[8] = {}; + + skel = stream__open_and_load(); + if (!ASSERT_OK_PTR(skel, "stream__open_and_load")) + return; + + prog_fd = bpf_program__fd(skel->progs.stream_syscall); + ret = bpf_prog_test_run_opts(prog_fd, &opts); + ASSERT_OK(ret, "ret"); + ASSERT_OK(opts.retval, "retval"); + + page_size = sysconf(_SC_PAGESIZE); + page = mmap(NULL, page_size * 2, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (!ASSERT_NEQ(page, MAP_FAILED, "mmap")) { + stream__destroy(skel); + return; + } + /* Leave only the first page mapped so a straddling copy faults. */ + if (!ASSERT_OK(munmap(page + page_size, page_size), "munmap second page")) { + munmap(page, page_size * 2); + stream__destroy(skel); + return; + } + + buf = page + page_size - 1; + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 3, NULL); + ASSERT_EQ(ret, 1, "partial bytes"); + ASSERT_EQ(buf[0], 'f', "first byte"); + + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, rest, sizeof(rest), NULL); + ASSERT_EQ(ret, 2, "remaining bytes"); + ASSERT_OK(memcmp(rest, "oo", 2), "remaining data"); + + munmap(page, page_size); + stream__destroy(skel); +} + static void test_address(struct bpf_program *prog, unsigned long *fault_addr_p) { LIBBPF_OPTS(bpf_test_run_opts, opts); diff --git a/tools/testing/selftests/bpf/progs/stream.c b/tools/testing/selftests/bpf/progs/stream.c index 8d8e53d37266..3d9c0a63fab8 100644 --- a/tools/testing/selftests/bpf/progs/stream.c +++ b/tools/testing/selftests/bpf/progs/stream.c @@ -36,7 +36,12 @@ struct { } array SEC(".maps"); #define ENOSPC 28 +#define E2BIG 7 #define _STR "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +#define _X64 "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +/* 1024 bytes: truncated by bstr_printf, must return -E2BIG. */ +#define _BIG_STR (_X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64 \ + _X64 _X64 _X64 _X64 _X64 _X64 _X64 _X64) int size; u64 fault_addr; @@ -118,6 +123,29 @@ int stream_syscall(void *ctx) return 0; } +SEC("syscall") +__success __retval(0) +int stream_oversize(void *ctx) +{ + int ret; + + ret = bpf_stream_printk(BPF_STDOUT, _BIG_STR); + if (ret != -E2BIG) + return ret ?: 1; + + /* The oversized output must not reduce the remaining stream capacity. */ + size = 0; + bpf_repeat(BPF_MAX_LOOPS) { + ret = bpf_stream_printk(BPF_STDOUT, _STR); + if (ret == -ENOSPC) + return size == 99954 ? 0 : 1; + if (ret) + return ret; + size += sizeof(_STR) - 1; + } + return 1; +} + SEC("syscall") __arch_x86_64 __arch_arm64 From 5e289c5a4a526e870efac0ca658635c47138593a Mon Sep 17 00:00:00 2001 From: Quanye Yang Date: Sat, 22 Aug 2026 14:55:33 +0800 Subject: [PATCH 28/90] bpf: Annotate bpf_obj_memcpy with data_race syzbot reported KCSAN write-write races when two tasks concurrently update the same map value. Both accesses reach the ordinary memcpy() paths in bpf_obj_memcpy() through copy_map_value(). Unlocked in-place updates of published map values are intentionally not serialized and may produce torn values. Callers requiring consistency must provide synchronization appropriate for the map type. bpf_long_memcpy() already annotates the same behavior for long-aligned copies. Annotate the ordinary memcpy() sites in bpf_obj_memcpy() with data_race(), matching bpf_long_memcpy(). This documents the existing concurrency semantics and suppresses KCSAN reports for these intentional races without changing synchronization or map update behavior. Reported-by: syzbot+44044637ef892e79ca2b@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=44044637ef892e79ca2b Signed-off-by: Quanye Yang Link: https://lore.kernel.org/bpf/20260822-bpf-kcsan-obj-memcpy-v3-1-4227b2edd8b4@proton.me Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ffa5626411ac..b3cd28d9e3b5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -572,7 +572,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (long_memcpy) bpf_long_memcpy(dst, src, size); else - memcpy(dst, src, size); + data_race(memcpy(dst, src, size)); return; } @@ -580,10 +580,10 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; - memcpy(dst + curr_off, src + curr_off, sz); + data_race(memcpy(dst + curr_off, src + curr_off, sz)); curr_off += rec->fields[i].size + sz; } - memcpy(dst + curr_off, src + curr_off, size - curr_off); + data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off)); } static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) From b4757714cc63a18c3850dfe99b3f3d1b67b14bf9 Mon Sep 17 00:00:00 2001 From: Chen Pei Date: Wed, 19 Aug 2026 20:09:23 +0800 Subject: [PATCH 29/90] bpf, riscv: Add support for signed arena loads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed loads from arena memory are currently rejected on riscv64, as bpf_jit_supports_insn() refuses BPF_MEMSX loads when in_arena is set, while x86 and arm64 gained support for them in v6.18. Compilers such as GCC-14 are free to generate signed loads into arena memory, which breaks loading of otherwise valid BPF programs on riscv64. Implement BPF_PROBE_MEM32SX support in the RV64 JIT by reusing the existing arena handling: the arena base (RV_REG_ARENA) is added to the source register and the load is emitted with sign extension (lb/lh/lw). Add BPF_PROBE_MEM32SX to the add_exception_handler() mode gate so that faulting loads get an exception table entry which clears the destination register and resumes execution. Since BPF_PROBE_MEM32SX shares its mode value (0xc0) with BPF_ATOMIC, the gate accepts it only for LDX class instructions so that plain atomic instructions do not register exception table entries. Verified by running the arena LDSX selftests (arena_ldsx_disasm, arena_ldsx_exception, arena_ldsx_s8/s16/s32) and the full arena_atomics test suite on riscv64 QEMU, all passing. Signed-off-by: Chen Pei Tested-by: Pu Lehui Reviewed-by: Pu Lehui Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/67422a421852c3eed2269c6def7b5569e727617f.1787140923.git.cp0613@linux.alibaba.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 47c7bf431ba8..e7378be171a9 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -747,6 +747,8 @@ static int add_exception_handler(const struct bpf_insn *insn, int dst_reg, if (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX && BPF_MODE(insn->code) != BPF_PROBE_MEM32 && + !(BPF_MODE(insn->code) == BPF_PROBE_MEM32SX && + BPF_CLASS(insn->code) == BPF_LDX) && BPF_MODE(insn->code) != BPF_PROBE_ATOMIC) return 0; @@ -1908,13 +1910,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + /* LDX | PROBE_MEM32SX: dst = *(signed size *)(src + RV_REG_ARENA + off) */ + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_B: + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32SX | BPF_W: { bool sign_ext; sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || - BPF_MODE(insn->code) == BPF_PROBE_MEMSX; + BPF_MODE(insn->code) == BPF_PROBE_MEMSX || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX; - if (BPF_MODE(insn->code) == BPF_PROBE_MEM32) { + if (BPF_MODE(insn->code) == BPF_PROBE_MEM32 || + BPF_MODE(insn->code) == BPF_PROBE_MEM32SX) { emit_add(RV_REG_T2, rs, RV_REG_ARENA, ctx); rs = RV_REG_T2; } @@ -2145,10 +2153,6 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) if (insn->imm == BPF_CMPXCHG) return rv_ext_enabled(ZACAS); break; - case BPF_LDX | BPF_MEMSX | BPF_B: - case BPF_LDX | BPF_MEMSX | BPF_H: - case BPF_LDX | BPF_MEMSX | BPF_W: - return false; } } From 8f2087a1b43bbc57231eca4b99415cc18fb3429a Mon Sep 17 00:00:00 2001 From: Chen Pei Date: Wed, 19 Aug 2026 20:09:24 +0800 Subject: [PATCH 30/90] selftests/bpf: Enable arena LDSX tests for riscv64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that the riscv64 JIT supports signed arena loads (BPF_PROBE_MEM32SX), enable the arena LDSX tests on riscv64: add JIT disassembly assertions for arena_ldsx_disasm (arena base in s7, add into t2, sign-extending lw/lh/lb loads) and run arena_ldsx_exception and arena_ldsx_s8/s16/s32 on riscv64. Signed-off-by: Chen Pei Tested-by: Pu Lehui Reviewed-by: Pu Lehui Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/1bab696ec36f196fa6ec74a0f056030b39730da7.1787140923.git.cp0613@linux.alibaba.com Signed-off-by: Kumar Kartikeya Dwivedi --- .../testing/selftests/bpf/progs/verifier_ldsx.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index 41340877dc9d..ed0a0f159bc1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -286,6 +286,19 @@ __jited("add x11, x0, x28") __jited("ldrsh x22, [x11, #0x18]") __jited("add x11, x0, x28") __jited("ldrsb x22, [x11, #0x20]") +__arch_riscv64 +__jited("add t2, a5, s7") +__jited("lw s3, 0x10(t2)") +__jited("add t2, a5, s7") +__jited("lh s3, 0x18(t2)") +__jited("add t2, a5, s7") +__jited("lb s3, 0x20(t2)") +__jited("add t2, a0, s7") +__jited("lw s4, 0x10(t2)") +__jited("add t2, a0, s7") +__jited("lh s4, 0x18(t2)") +__jited("add t2, a0, s7") +__jited("lb s4, 0x20(t2)") __naked void arena_ldsx_disasm(void *ctx) { asm volatile ( @@ -317,6 +330,7 @@ __description("Arena LDSX Exception") __success __retval(0) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_exception(void *ctx) { asm volatile ( @@ -338,6 +352,7 @@ __description("Arena LDSX, S8") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s8(void *ctx) { asm volatile ( @@ -369,6 +384,7 @@ __description("Arena LDSX, S16") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s16(void *ctx) { asm volatile ( @@ -400,6 +416,7 @@ __description("Arena LDSX, S32") __success __retval(-1) __arch_x86_64 __arch_arm64 +__arch_riscv64 __naked void arena_ldsx_s32(void *ctx) { asm volatile ( From ce36e38beb6863664994f5676122359c8d323203 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 24 Aug 2026 21:57:46 +0200 Subject: [PATCH 31/90] selftests/bpf: Resolve bpftool through bpftool_helpers in core_reloc run_btfgen() shells out to a hardcoded "./bpftool". That path only exists in the build tree, where DEFINE_TEST_RUNNER_RULES symlinks bpftool next to each runner binary. After "make install" it is not there: commit 0b236ac75d04 ("selftests/bpf: Fix make install target") dropped bpftool from TEST_GEN_PROGS_EXTENDED, which used to flatten it into $(INSTALL_PATH), and installs it under $(INSTALL_PATH)/tools/sbin/ instead, matching detect_bpftool_path(). Running test_progs from the install tree therefore fails all 64 core_reloc_btfgen subtests: run_core_reloc_tests:FAIL:run_btfgen unexpected error: 32512 (errno 95) sh: line 1: ./bpftool: No such file or directory 32512 is 127 << 8, i.e. the shell could not find the binary. Use run_bpftool_command() from bpftool_helpers.h, which locates bpftool via $BPFTOOL, ./tools/sbin/bpftool or ../tools/sbin/bpftool, the last of which also covers the test_progs flavors. bpftool_helpers.c is already in TRUNNER_EXTRA_SOURCES, so no build change is needed. Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Jiri Olsa Link: https://lore.kernel.org/bpf/20260824195746.3070450-1-memxor@gmail.com --- tools/testing/selftests/bpf/prog_tests/core_reloc.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index 08963c82f30b..80432c4656b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -7,6 +7,7 @@ #include #include #include +#include "bpftool_helpers.h" static int duration = 0; @@ -985,16 +986,15 @@ static size_t roundup_page(size_t sz) static int run_btfgen(const char *src_btf, const char *dst_btf, const char *objpath) { - char command[4096]; + char args[4096]; int n; - n = snprintf(command, sizeof(command), - "./bpftool gen min_core_btf %s %s %s", + n = snprintf(args, sizeof(args), "gen min_core_btf %s %s %s", src_btf, dst_btf, objpath); - if (n < 0 || n >= sizeof(command)) + if (n < 0 || n >= sizeof(args)) return -1; - return system(command); + return run_bpftool_command(args); } static void run_core_reloc_tests(bool use_btfgen) From ea0b60fae724aa1147d4a07eda5131d3a9a9977e Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 23 Aug 2026 21:32:00 +0200 Subject: [PATCH 32/90] bpf: Reject oversized stream read buffers BPF_PROG_STREAM_READ_BY_FD exposes stream_buf_len as a u32, but the internal stream reader accepts an int. Values above INT_MAX therefore become negative before bpf_stream_read(), producing negative consumption lengths. The generic usercopy size check prevents the oversized copy, but emits a warning and reports EFAULT for an unchecked argument. Change bpf_prog_stream_read() to accept u32 and reject values that the signed reader cannot represent. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260823193202.2389715-2-memxor@gmail.com --- include/linux/bpf.h | 2 +- kernel/bpf/stream.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b3cd28d9e3b5..c2027688be3e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4101,7 +4101,7 @@ void bpf_put_buffers(void); void bpf_prog_stream_init(struct bpf_prog *prog); void bpf_prog_stream_free(struct bpf_prog *prog); -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 7a5c3ac8676b..2b80a0599865 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -203,13 +203,15 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) return ret ? ret : len - rem_len; } -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len) { struct bpf_stream *stream; stream = bpf_stream_get(stream_id, prog->aux); if (!stream) return -ENOENT; + if (len > INT_MAX) + return -EINVAL; return bpf_stream_read(stream, buf, len); } From 05ea1b6e2a5d008c24b7a2c6eaa3f708d9026994 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 23 Aug 2026 21:32:01 +0200 Subject: [PATCH 33/90] selftests/bpf: Cover oversized stream read buffers Exercise BPF_PROG_STREAM_READ_BY_FD with a UINT_MAX buffer length and require EINVAL. Continue with the existing two-part read of the queued payload to verify that rejecting the oversized request does not consume or otherwise disturb stream data. Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260823193202.2389715-3-memxor@gmail.com --- tools/testing/selftests/bpf/prog_tests/stream.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/stream.c b/tools/testing/selftests/bpf/prog_tests/stream.c index 4d8054a23cc7..74bd15c4bfba 100644 --- a/tools/testing/selftests/bpf/prog_tests/stream.c +++ b/tools/testing/selftests/bpf/prog_tests/stream.c @@ -46,6 +46,10 @@ void test_stream_syscall(void) ret = -errno; ASSERT_EQ(ret, -EFAULT, "bad stream buf"); + ASSERT_LT(bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, UINT_MAX, NULL), 0, "error"); + ret = -errno; + ASSERT_EQ(ret, -EINVAL, "large stream buf"); + ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 2, NULL); ASSERT_EQ(ret, 2, "bytes"); ret = bpf_prog_stream_read(prog_fd, BPF_STREAM_STDOUT, buf, 2, NULL); From 8b94878b2572f0f41e6002d0fe2cc4086ddc2a47 Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 25 Aug 2026 23:04:33 +0800 Subject: [PATCH 34/90] libbpf: Fix usdt attach failure when nop10 crosses page boundary The kernel refuses to attach to a nop10 that crosses a page boundary, since it can't be atomically rewritten: /* can_optimize(), arch/x86/kernel/uprobes.c */ /* We can't do cross page atomic writes yet. */ return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE; Whether the nop10 crosses a page is purely up to the binary layout, so this does happen in practice. libbpf doesn't check for it and blindly shifts the uprobe onto the nop10, and the attach then fails with -ENOTSUPP. Just keep the uprobe on the preceding 1-byte nop in that case, it works everywhere as a regular int3 uprobe. Fixes: ee2862439e5c ("libbpf: Change has_nop_combo to work on top of nop10") Signed-off-by: Jiayuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260825150444.31603-1-jiayuan.chen@linux.dev --- tools/lib/bpf/usdt.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index 2e56e3ab5b6c..ee9d1b614883 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -608,8 +608,12 @@ static bool has_nop_combo(int fd, long off) unsigned char nop_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, }; + long page_sz = getpagesize(); unsigned char buf[11]; + /* the kernel can't attach to a nop10 that crosses a page boundary */ + if ((off + 1) % page_sz + 10 > page_sz) + return false; if (pread(fd, buf, 11, off) != 11) return false; return memcmp(buf, nop_combo, 11) == 0; From 1555de33018fad06ca5dfa2fad5d3d1c6a6eb2fe Mon Sep 17 00:00:00 2001 From: Jiayuan Chen Date: Tue, 25 Aug 2026 23:04:34 +0800 Subject: [PATCH 35/90] selftests/bpf: Add test for usdt probe with page-crossing nop10 Add a USDT probe laid out so that its nop10 crosses a page boundary and check that attaching to it succeeds and the probe fires. Without the previous fix libbpf shifts the uprobe onto the nop10 and the attach fails with -ENOTSUPP. Signed-off-by: Jiayuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260825150444.31603-2-jiayuan.chen@linux.dev --- tools/testing/selftests/bpf/prog_tests/usdt.c | 54 +++++++++++++++++++ tools/testing/selftests/bpf/usdt_2.c | 16 ++++++ 2 files changed, 70 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index 8004c9568ffa..eff1e57ab13c 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c @@ -250,6 +250,7 @@ static void subtest_basic_usdt(bool optimized) #ifdef __x86_64__ extern void usdt_1(void); extern void usdt_2(void); +extern void usdt_2_cross_page(void); extern void usdt_red_zone_trigger(void); static unsigned char nop1[1] = { 0x90 }; @@ -342,6 +343,57 @@ static void subtest_optimized_attach(void) test_usdt__destroy(skel); } +/* + * Test attachment to a USDT probe whose nop10 crosses a page boundary. + * The kernel can't optimize such nop10, so libbpf keeps the uprobe on + * the preceding 1-byte nop. Do not assume any particular placement + * here, though: however the probe ends up attached, the attachment + * must succeed and the probe must fire. + */ +static void subtest_optimized_attach_cross_page(void) +{ + long page_sz = sysconf(_SC_PAGESIZE); + struct test_usdt *skel; + __u8 *addr = NULL; + long i; + + /* combo is placed up to a page of padding after the function start */ + for (i = 0; i < 2 * page_sz; i++) { + if (!memcmp((void *)usdt_2_cross_page + i, nop1_nop10_combo, 11)) { + addr = (void *)usdt_2_cross_page + i; + break; + } + } + if (!ASSERT_OK_PTR(addr, "find_nop1_nop10_combo")) + return; + + /* layout sanity check: the nop10 must cross the page boundary */ + if (!ASSERT_GT((unsigned long)(addr + 1) % page_sz + 10, page_sz, + "nop10_crosses_page")) + return; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "test_usdt__open_and_load")) + return; + + skel->bss->my_pid = getpid(); + + skel->links.usdt0 = bpf_program__attach_usdt(skel->progs.usdt0, + 0 /*self*/, "/proc/self/exe", + "optimized_attach", + "usdt_2_cross_page", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt0, "bpf_program__attach_usdt")) + goto cleanup; + + usdt_2_cross_page(); + usdt_2_cross_page(); + + ASSERT_EQ(skel->bss->usdt0_called, 2, "usdt0_called"); + +cleanup: + test_usdt__destroy(skel); +} + /* * Test that USDT arguments survive nop10 optimization in a function where * the compiler places operands in the red zone. @@ -660,6 +712,8 @@ void test_usdt(void) subtest_basic_usdt(true); if (test__start_subtest("optimized_attach")) subtest_optimized_attach(); + if (test__start_subtest("optimized_attach_cross_page")) + subtest_optimized_attach_cross_page(); if (test__start_subtest("optimized_red_zone")) subtest_optimized_red_zone(); #endif diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c index 5e38f8605b02..3b7024b9b08b 100644 --- a/tools/testing/selftests/bpf/usdt_2.c +++ b/tools/testing/selftests/bpf/usdt_2.c @@ -13,6 +13,22 @@ void usdt_2(void) USDT(optimized_attach, usdt_2); } +/* + * Force the nop1,nop10 combo of the USDT probe to a spot where the nop10 + * crosses a page boundary: .balign starts the padding exactly at a page + * start regardless of the compiler-generated prologue size, and the 4086 + * one-byte nops put the nop1 at page offset 4086, so the following nop10 + * occupies the last 9 bytes of that page and 1 byte of the next one. + * The kernel can't optimize such nop10, so libbpf must keep the uprobe + * on the 1-byte nop. + */ +__attribute__((noinline)) +void usdt_2_cross_page(void) +{ + asm volatile (".balign 4096, 0x90\n\t.skip 4086, 0x90"); + USDT(optimized_attach, usdt_2_cross_page); +} + static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF; static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE; static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE; From b0eeb9e0f548ab4b555b588a8cbb09de7a690c47 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Mon, 24 Aug 2026 14:01:40 -0700 Subject: [PATCH 36/90] selftests/bpf: Name the remaining placeholder programs dummy_test [NFC] Thirteen of the fifteen feature-gated placeholder programs are called dummy_test. Two are not: hence this patch. This is purely cosmetic as nothing depends on those names. The non-compliant setups use RUN_TESTS(skel) which goes through all programs normally. This is just more consistent and grep friendly. Signed-off-by: Vineet Gupta Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260824210142.3933580-2-vineet.gupta@linux.dev --- tools/testing/selftests/bpf/progs/arena_kfunc.c | 2 +- tools/testing/selftests/bpf/progs/stack_arg_fail.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index bf0d304e0e59..47bed5c4c488 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -234,7 +234,7 @@ __arch_x86_64 __arch_arm64 __description("arena_arg_stack: not supported, dummy test") __success -int arena_arg_stack(void *ctx) +int dummy_test(void *ctx) { return 0; } diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c index ad9d4bfe15dc..6f6f59306d0e 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_fail.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c @@ -104,7 +104,7 @@ __naked void r11_store_zero_off(void) SEC("tc") __description("stack_arg_fail: not supported, dummy test") __success -int test_stack_arg_big(struct __sk_buff *skb) +int dummy_test(struct __sk_buff *skb) { return 0; } From 8928f5ffdb65f0f69babb006123d256bd774f2d3 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Mon, 24 Aug 2026 14:01:41 -0700 Subject: [PATCH 37/90] selftests/bpf: Report placeholder tests as SKIP, not OK Several verifier test files wrap their contents in a feature gate and provide a one-line placeholder program in the #else arm, so the file still yields something when the toolchain or JIT cannot support the tested instructions. However the placeholder is annotated __success, so it reports as a pass, whereas this is really a SKIP. | #318/1 verifier_ldsx/cpuv4 is not supported by compiler or jit, ...:OK | #318 verifier_ldsx:OK This causes skipped coverage to be overlooked: see the next patch where BPF_GCC builds miss a whole bunch of test due to clang only gate. Add a __skip("reason") annotation for a compile-time gate, and use it for the placeholders. After this change, above tests reported as | #318/1 verifier_ldsx/cpuv4 is not supported by compiler or jit, ...:SKIP | #318 verifier_ldsx:SKIP Signed-off-by: Vineet Gupta Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260824210142.3933580-3-vineet.gupta@linux.dev --- tools/testing/selftests/bpf/progs/arena_kfunc.c | 1 + tools/testing/selftests/bpf/progs/bpf_misc.h | 6 ++++++ tools/testing/selftests/bpf/progs/stack_arg_fail.c | 1 + tools/testing/selftests/bpf/progs/stack_arg_precision.c | 1 + .../testing/selftests/bpf/progs/verifier_aggregate_ret.c | 1 + tools/testing/selftests/bpf/progs/verifier_bswap.c | 1 + tools/testing/selftests/bpf/progs/verifier_gotol.c | 1 + tools/testing/selftests/bpf/progs/verifier_ldsx.c | 1 + .../testing/selftests/bpf/progs/verifier_load_acquire.c | 1 + tools/testing/selftests/bpf/progs/verifier_movsx.c | 1 + tools/testing/selftests/bpf/progs/verifier_percpu_addr.c | 1 + .../testing/selftests/bpf/progs/verifier_private_stack.c | 1 + tools/testing/selftests/bpf/progs/verifier_sdiv.c | 1 + tools/testing/selftests/bpf/progs/verifier_stack_arg.c | 1 + .../selftests/bpf/progs/verifier_stack_arg_order.c | 1 + .../testing/selftests/bpf/progs/verifier_store_release.c | 1 + tools/testing/selftests/bpf/test_loader.c | 9 +++++++++ 17 files changed, 30 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/arena_kfunc.c b/tools/testing/selftests/bpf/progs/arena_kfunc.c index 47bed5c4c488..50609f3b0564 100644 --- a/tools/testing/selftests/bpf/progs/arena_kfunc.c +++ b/tools/testing/selftests/bpf/progs/arena_kfunc.c @@ -233,6 +233,7 @@ SEC("syscall") __arch_x86_64 __arch_arm64 __description("arena_arg_stack: not supported, dummy test") +__skip("arena_arg_stack: not supported") __success int dummy_test(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 5eacf1b43252..88b0bfba83bb 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -106,6 +106,11 @@ * __description Text to be used for display and as an additional filter * alias, while the original program name stays matchable. * + * __skip Report the test as SKIP with the given reason instead of + * running it. For placeholder programs that stand in for a + * feature the toolchain or JIT cannot provide, so a run does + * not report OK for coverage it never executed. + * * __log_level Log level to use for the program, numeric value expected. * * __flag Adds one flag use for the program, the following values are valid: @@ -139,6 +144,7 @@ #define __failure __test_tag("test_expect_failure") #define __success __test_tag("test_expect_success") #define __description(desc) __test_tag("test_description=" desc) +#define __skip(reason) __test_tag("test_skip=" reason) #define __msg_unpriv(msg) __test_tag("test_expect_msg_unpriv=" msg) #define __not_msg_unpriv(msg) __test_tag("test_expect_not_msg_unpriv=" msg) #define __xlated_unpriv(msg) __test_tag("test_expect_xlated_unpriv=" msg) diff --git a/tools/testing/selftests/bpf/progs/stack_arg_fail.c b/tools/testing/selftests/bpf/progs/stack_arg_fail.c index 6f6f59306d0e..eed97d582515 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_fail.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_fail.c @@ -103,6 +103,7 @@ __naked void r11_store_zero_off(void) SEC("tc") __description("stack_arg_fail: not supported, dummy test") +__skip("stack_arg_fail: not supported") __success int dummy_test(struct __sk_buff *skb) { diff --git a/tools/testing/selftests/bpf/progs/stack_arg_precision.c b/tools/testing/selftests/bpf/progs/stack_arg_precision.c index bee2eeec021d..ce0301a41fa9 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_precision.c @@ -124,6 +124,7 @@ __naked void stack_arg_precision_bpf2bpf(void) SEC("socket") __description("stack_arg_precision: not supported, dummy test") +__skip("stack_arg_precision: not supported") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c index 7851bade2b40..f5c1166027a1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c @@ -167,6 +167,7 @@ int aggregate_ret_kfunc_struct_c_test(struct __sk_buff *skb) SEC("socket") __description("verifier_aggregate_ret: needs LLVM 23, dummy test") +__skip("needs LLVM 23") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_bswap.c b/tools/testing/selftests/bpf/progs/verifier_bswap.c index cffaf36192bc..074faef5480e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bswap.c +++ b/tools/testing/selftests/bpf/progs/verifier_bswap.c @@ -117,6 +117,7 @@ l0_%=: \ SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_gotol.c b/tools/testing/selftests/bpf/progs/verifier_gotol.c index d5d8f24df394..c0fd30a3a4c1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_gotol.c +++ b/tools/testing/selftests/bpf/progs/verifier_gotol.c @@ -52,6 +52,7 @@ __naked void gotol_large_imm(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index ed0a0f159bc1..5c704817111f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -453,6 +453,7 @@ void kfunc_root(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c index d17026d7480d..bcc34e798c63 100644 --- a/tools/testing/selftests/bpf/progs/verifier_load_acquire.c +++ b/tools/testing/selftests/bpf/progs/verifier_load_acquire.c @@ -267,6 +267,7 @@ __naked void load_acquire_with_invalid_reg(void) SEC("socket") __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test") +__skip("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_movsx.c b/tools/testing/selftests/bpf/progs/verifier_movsx.c index a4d8814eb5ed..5559c0583dd1 100644 --- a/tools/testing/selftests/bpf/progs/verifier_movsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_movsx.c @@ -343,6 +343,7 @@ label_%=: \ SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c index 967f4e6e3a49..fd418f27996f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c +++ b/tools/testing/selftests/bpf/progs/verifier_percpu_addr.c @@ -61,6 +61,7 @@ __naked void percpu_addr(void) SEC("raw_tp") __description("percpu addr dummy") +__skip("percpu addr tests need x86") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index ea0a7e73331d..3d9994a52464 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -377,6 +377,7 @@ int private_stack_max_depth(void) SEC("kprobe") __description("private stack is not supported, use a dummy test") +__skip("private stack is not supported") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_sdiv.c b/tools/testing/selftests/bpf/progs/verifier_sdiv.c index 95f3239ce228..515644ace516 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sdiv.c +++ b/tools/testing/selftests/bpf/progs/verifier_sdiv.c @@ -1271,6 +1271,7 @@ __naked void smod32_int_min_mod_neg2_imm(void) SEC("socket") __description("cpuv4 is not supported by compiler or jit, use a dummy test") +__skip("cpuv4 is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index 7e0ce5db28a0..51d22faf4559 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -436,6 +436,7 @@ __naked void stack_arg_sequential_calls(void) SEC("socket") __description("stack_arg is not supported by compiler or jit, use a dummy test") +__skip("stack_arg is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index c9fe4857da3f..8e4325273b3d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -174,6 +174,7 @@ __naked void stack_arg_read_without_write_2(void) SEC("socket") __description("stack_arg order is not supported by compiler or jit, use a dummy test") +__skip("stack_arg order is not supported by compiler or jit") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_store_release.c b/tools/testing/selftests/bpf/progs/verifier_store_release.c index 72f1eb006074..0abbee5ce109 100644 --- a/tools/testing/selftests/bpf/progs/verifier_store_release.c +++ b/tools/testing/selftests/bpf/progs/verifier_store_release.c @@ -290,6 +290,7 @@ __naked void store_release_with_invalid_reg(void) SEC("socket") __description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test") +__skip("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release") __success int dummy_test(void) { diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 07807757b518..794a7dfb0579 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -70,6 +70,7 @@ struct test_spec { int arch_mask; int load_mask; int linear_sz; + const char *skip_reason; bool auxiliary; bool valid; }; @@ -456,6 +457,8 @@ static int parse_test_spec(struct test_loader *tester, continue; if ((val = str_has_pfx(s, "test_description="))) { description = val; + } else if ((val = str_has_pfx(s, "test_skip="))) { + spec->skip_reason = val; } else if (strcmp(s, "test_expect_failure") == 0) { spec->priv.expect_failure = true; spec->mode_mask |= PRIV; @@ -1327,6 +1330,12 @@ void run_subtest(struct test_loader *tester, if (!test__start_subtest_with_desc(subspec->name, subspec->description)) return; + if (spec->skip_reason) { + printf("%s:SKIP: %s\n", __func__, spec->skip_reason); + test__skip(); + return; + } + if ((get_current_arch() & spec->arch_mask) == 0) { test__skip(); return; From 48b69cc8825d411a4c4df32de854434e9235a1b0 Mon Sep 17 00:00:00 2001 From: Vineet Gupta Date: Mon, 24 Aug 2026 14:01:42 -0700 Subject: [PATCH 38/90] selftests/bpf: Enable some of the blocked cpuv4 instruction tests for bpf-gcc A bunch of tests are gated behind clang only __clang_major__ and thus fail to build under BPF_GCC. What's worse is, before the prev patch they were also declared as passing/OK. This need not be because (a) bpf-gcc does support the feature in codegen and (b) also advertises correctly with feature support preprocessor macros: __BPF_FEATURE_BSWAP, __BPF_FEATURE_GOTOL, __BPF_FEATURE_LDSX, __BPF_FEATURE_MOVSX, __BPF_FEATURE_SDIV_SMOD, __BPF_FEATURE_ST So loosen up the gates to also add specific feature check. clang >=18 has to stay, despite it being true for so long: clang only defines __BPF_FEATURE_MOVSX and friends at -mcpu=v4, but assembles the inline asm at -mcpu=v3 too, and dropping it would remove these tests from the default test_progs flavour where they run today. Since bpf-gcc doesn't support addr_space_cast, wrap the corresponding tests under __BPF_FEATURE_ADDR_SPACE_CAST to skip under it. And also move the map under that guard so that a bpf-gcc object does not declare an arena map that no program in it references. Recovered under BPF_GCC (programs per object, was 1 dummy each): verifier_sdiv 80 verifier_movsx 17 verifier_ldsx 14 (20 under clang; 5 arena tests, kfunc_root and the arena map stay clang-only) verifier_bswap 13 CAN_USE_GOTOL and CAN_USE_BPF_ST only gate parts of a file, so those were not dummies to begin with: compute_live_registers 17 -> 19 (21 under clang) verifier_gotol 1 -> 2 verifier_iterating_callbacks 34 -> 35 The clang builds are unchanged: verifier_ldsx still emits 20 programs under the cpuv4 flavour. One thing left alone: verifier_load_acquire.c and verifier_store_release.c stay clang-only, and their gates are correct as gas doesn't implement those (experimental -mxbpf not implemented at all). FWIW LLVM groups those under -mcpu=v4 and defines __BPF_FEATURE_LOAD_ACQ_STORE_REL there. Signed-off-by: Vineet Gupta Signed-off-by: Daniel Borkmann Acked-by: Eduard Zingerman Link: https://lore.kernel.org/bpf/20260824210142.3933580-4-vineet.gupta@linux.dev --- tools/testing/selftests/bpf/progs/bpf_misc.h | 4 ++-- .../bpf/progs/compute_live_registers.c | 8 ++++++++ .../selftests/bpf/progs/verifier_bswap.c | 2 +- .../selftests/bpf/progs/verifier_ldsx.c | 18 +++++++++++------- .../selftests/bpf/progs/verifier_movsx.c | 2 +- .../selftests/bpf/progs/verifier_sdiv.c | 2 +- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index 88b0bfba83bb..eb88d9ce6c34 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -261,11 +261,11 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_GOTOL)) #define CAN_USE_GOTOL #endif -#if __clang_major__ >= 18 +#if __clang_major__ >= 18 || defined(__BPF_FEATURE_ST) #define CAN_USE_BPF_ST #endif diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index 338e67cd8856..941b65fe7c07 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -299,7 +299,15 @@ __naked void gotol(void) "r3 = 24;" "if r1 > 0x7 goto +2;" "r0 = r2;" +#ifdef __clang__ "gotol +1;" +#else + /* + * gas mis-parses 'gotol +1' as 'goto l+1', same encoding + * without the sign (binutils PR gas/34558). + */ + "gotol 1;" +#endif "r0 = r3;" "exit;" : diff --git a/tools/testing/selftests/bpf/progs/verifier_bswap.c b/tools/testing/selftests/bpf/progs/verifier_bswap.c index 074faef5480e..48ffb5b242d2 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bswap.c +++ b/tools/testing/selftests/bpf/progs/verifier_bswap.c @@ -8,7 +8,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_BSWAP)) SEC("socket") __description("BSWAP, 16") diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index 5c704817111f..7e25e1b8cd0b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -9,13 +9,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 1); -} arena SEC(".maps"); + (__clang_major__ >= 18 || defined(__BPF_FEATURE_LDSX)) SEC("socket") __description("LDSX, S8") @@ -263,6 +257,14 @@ __naked void ldsx_ctx_8(void) : __clobber_all); } +#ifdef __BPF_FEATURE_ADDR_SPACE_CAST + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1); +} arena SEC(".maps"); + SEC("syscall") __description("Arena LDSX Disasm") __success @@ -449,6 +451,8 @@ void kfunc_root(void) bpf_arena_alloc_pages(0, 0, 0, 0, 0); } +#endif /* __BPF_FEATURE_ADDR_SPACE_CAST */ + #else SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/verifier_movsx.c b/tools/testing/selftests/bpf/progs/verifier_movsx.c index 5559c0583dd1..195b27a51224 100644 --- a/tools/testing/selftests/bpf/progs/verifier_movsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_movsx.c @@ -8,7 +8,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_MOVSX)) SEC("socket") __description("MOV32SX, S8") diff --git a/tools/testing/selftests/bpf/progs/verifier_sdiv.c b/tools/testing/selftests/bpf/progs/verifier_sdiv.c index 515644ace516..d6cbe1e27fd6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_sdiv.c +++ b/tools/testing/selftests/bpf/progs/verifier_sdiv.c @@ -9,7 +9,7 @@ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \ defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \ defined(__TARGET_ARCH_loongarch)) && \ - __clang_major__ >= 18 + (__clang_major__ >= 18 || defined(__BPF_FEATURE_SDIV_SMOD)) SEC("socket") __description("SDIV32, non-zero imm divisor, check 1") From 713c02b59e21840f623a3e31c43d967026c75181 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 24 Aug 2026 17:26:56 +0800 Subject: [PATCH 39/90] bpftool: Fix spurious batch file read error do_batch() checks errno after the read loop to detect read failures, but fgets() does not clear errno on success, so a stale errno left by a previously executed command (e.g. map dump's EBADF from a double close) makes bpftool report a batch file read failure and exit with an error even though every command succeeded. Clear errno before each fgets() call, so the post-loop check only sees the outcome of the last read: zero on success or EOF, E2BIG for an overlong line, and a genuine errno when fgets() fails. Since errno is now reset before every read in batch mode, drop the USE_LIBCAP errno reset in main() that existed only to keep errno clean for the batch mode. Fixes: 71bb428fe2c1 ("tools: bpf: add bpftool") Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260824092657.1789956-2-chenyuan_fl@163.com --- tools/bpf/bpftool/main.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index c91e1a6e1a1e..7a0c214f08a0 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -365,7 +365,11 @@ static int do_batch(int argc, char **argv) if (json_output) jsonw_start_array(json_wtr); - while (fgets(buf, sizeof(buf), fp)) { + for (;;) { + errno = 0; + if (!fgets(buf, sizeof(buf), fp)) + break; + cp = strchr(buf, '#'); if (cp) *cp = '\0'; @@ -467,16 +471,6 @@ int main(int argc, char **argv) setlinebuf(stdout); -#ifdef USE_LIBCAP - /* Libcap < 2.63 hooks before main() to compute the number of - * capabilities of the running kernel, and doing so it calls prctl() - * which may fail and set errno to non-zero. - * Let's reset errno to make sure this does not interfere with the - * batch mode. - */ - errno = 0; -#endif - last_do_help = do_help; pretty_output = false; json_output = false; From ac3d88577cdaa5330cf32d03ee3808df8fa338ac Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 24 Aug 2026 17:26:57 +0800 Subject: [PATCH 40/90] bpftool: Fix bypass of the batch line length check by comments do_batch() strips trailing comments by truncating the line at '#' before checking whether fgets() filled the buffer. If a batch line longer than the buffer contains a '#' within the first sizeof(buf) - 1 bytes, the truncation makes strlen(buf) smaller and the line-length check is bypassed. The unread remainder of the line then stays in the file stream and is parsed and executed as a separate command on the next loop iteration. Continuation lines handled below are affected the same way: an overlong continuation line containing '#' bypasses the "command is too long" check, and its unread remainder is executed as a separate command. Move the line-length checks before the comment is stripped, so they see the full line as read from the file and overlong lines are rejected regardless of comments. A line that fills the buffer exactly is now rejected as well, which is fine: batch command lines are not expected to come anywhere near the buffer limit. Fixes: 71bb428fe2c1 ("tools: bpf: add bpftool") Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260824092657.1789956-3-chenyuan_fl@163.com --- tools/bpf/bpftool/main.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 7a0c214f08a0..5ababd8f7d0a 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -370,15 +370,15 @@ static int do_batch(int argc, char **argv) if (!fgets(buf, sizeof(buf), fp)) break; - cp = strchr(buf, '#'); - if (cp) - *cp = '\0'; - if (strlen(buf) == sizeof(buf) - 1) { errno = E2BIG; break; } + cp = strchr(buf, '#'); + if (cp) + *cp = '\0'; + /* Append continuation lines if any (coming after a line ending * with '\' in the batch file). */ @@ -391,15 +391,15 @@ static int do_batch(int argc, char **argv) goto err_close; } - cp = strchr(contline, '#'); - if (cp) - *cp = '\0'; - if (strlen(buf) + strlen(contline) + 1 > sizeof(buf)) { p_err("command %u is too long", lines); err = -1; goto err_close; } + + cp = strchr(contline, '#'); + if (cp) + *cp = '\0'; buf[strlen(buf) - 2] = '\0'; strcat(buf, contline); } From 5e875ae9e9ffd5c76c6bdf8982f99de340dd46af Mon Sep 17 00:00:00 2001 From: Leon Hwang Date: Wed, 26 Aug 2026 22:56:13 +0800 Subject: [PATCH 41/90] selftests/bpf: Avoid flaky resize value test for percpu data The bpf_map__set_value_size() test for percpu data relies on the tail 'cpu_id' field in the ".percpu" map. The test would fail, when 'cpu_id' is not the last field in the map. To avoid the flaky test, factor out a subtest to use a dedicated ".percpu.arr" map that only has one field 'int arr[1]'. And, skip the subtest when fail to load skel with errno E2BIG and the new value size is larger than ALIGN(32KiB, PAGE_SIZE). And, drop 'args' in test_percpu_data_on_cpus(), which isn't used in bpf prog side. And, enhance the test to cover 'set' and 'nums[6]'. Fixes: 4c9241bd731a ("selftests/bpf: Add tests to verify global percpu data") Signed-off-by: Leon Hwang Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260826145613.34182-1-leon.hwang@linux.dev --- .../bpf/prog_tests/global_data_init.c | 96 ++++++++++++++----- .../bpf/progs/test_global_percpu_data.c | 10 +- 2 files changed, 80 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 5671c31085cd..7c539cbcf3a1 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -68,12 +68,9 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f { struct test_global_percpu_data__percpu *data = NULL; int i, err, key = 0, num_online, run = 0; - __u64 args[2] = {0x1234ULL, 0x5678ULL}; size_t data_sz; bool *online; LIBBPF_OPTS(bpf_test_run_opts, topts, - .ctx_in = args, - .ctx_size_in = sizeof(args), .flags = BPF_F_TEST_RUN_ON_CPU, ); @@ -81,7 +78,7 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f if (!ASSERT_OK(err, "parse_cpu_mask_file")) return; - data_sz = map ? bpf_map__value_size(map) : sizeof(*data); + data_sz = sizeof(*data); data = calloc(1, data_sz); if (!ASSERT_OK_PTR(data, "calloc percpu data")) goto out; @@ -109,7 +106,7 @@ static void test_percpu_data_on_cpus(struct bpf_map *map, int map_fd, int prog_f break; ASSERT_EQ(*runp, ++run, "run"); - ASSERT_EQ(data->cpu_id[0], i, "cpu_id"); + ASSERT_EQ(data->cpu_id, i, "cpu_id"); ASSERT_EQ(data->data, 1, "data"); ASSERT_TRUE(data->set, "set"); ASSERT_EQ(data->nums[6], 0xc0de, "nums[6]"); @@ -127,7 +124,6 @@ static void test_global_percpu_data_init(void) { struct test_global_percpu_data__percpu init_value = {}; struct test_global_percpu_data__percpu *init_data; - const __u32 desired_sz = sysconf(_SC_PAGE_SIZE); struct test_global_percpu_data *skel = NULL; size_t init_data_sz; struct bpf_map *map; @@ -163,9 +159,11 @@ static void test_global_percpu_data_init(void) if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) goto out; + init_value.set = 1; init_value.data = 2; init_value.nums[6] = -1; init_value.struct_data.i = 2; + init_value.struct_data.set = 1; init_value.struct_data.nums[6] = -1; err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); if (!ASSERT_OK(err, "bpf_map__set_initial_value")) @@ -177,35 +175,24 @@ static void test_global_percpu_data_init(void) ASSERT_EQ(init_data->data, init_value.data, "init_value data"); ASSERT_EQ(init_data->set, init_value.set, "init_value set"); + ASSERT_EQ(init_data->nums[6], init_value.nums[6], "init_value nums[6]"); ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.set, init_value.struct_data.set, + "init_value struct_data.set"); ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], "init_value struct_data.nums[6]"); ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); ASSERT_EQ(skel->percpu->set, init_value.set, "skel->percpu->set"); + ASSERT_EQ(skel->percpu->nums[6], init_value.nums[6], "skel->percpu->nums[6]"); ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.set, init_value.struct_data.set, + "skel->percpu->struct_data.set"); ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], "skel->percpu->struct_data.nums[6]"); - ASSERT_GT(desired_sz, sizeof(init_value), "desired_sz"); - err = bpf_map__set_value_size(map, desired_sz); - if (!ASSERT_OK(err, "bpf_map__set_value_size")) - goto out; - if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "percpu value size")) - goto out; - if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "percpu BTF value type")) - goto out; - - init_data = bpf_map__initial_value(map, &init_data_sz); - if (!ASSERT_OK_PTR(init_data, "resized bpf_map__initial_value")) - goto out; - if (!ASSERT_EQ(init_data_sz, desired_sz, "resized initial value size")) - goto out; - if (!ASSERT_EQ(init_data->data, init_value.data, "resized initial value data")) - goto out; - err = test_global_percpu_data__load(skel); if (!ASSERT_OK(err, "test_global_percpu_data__load")) goto out; @@ -236,6 +223,67 @@ static void test_global_percpu_data_lskel(void) test_global_percpu_data_lskel__destroy(lskel); } +#define PCPU_MIN_UNIT_SIZE (32 << 10) + +static void test_global_percpu_data_map_resize(void) +{ + const __u32 page_size = sysconf(_SC_PAGE_SIZE); + const __u32 pcpu_max_value_size = ALIGN(PCPU_MIN_UNIT_SIZE, page_size); + const __u32 desired_sz = page_size * 2; + struct test_global_percpu_data *skel; + const int arr_value = -1; + struct bpf_map *map; + int err, prog_fd; + size_t data_sz; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + map = skel->maps.percpu_arr; + skel->percpu_arr->arr[0] = arr_value; + err = bpf_map__set_value_size(map, desired_sz); + if (!ASSERT_OK(err, "bpf_map__set_value_size")) + goto out; + if (!ASSERT_EQ(bpf_map__value_size(map), desired_sz, "bpf_map__value_size")) + goto out; + if (!ASSERT_NEQ(bpf_map__btf_value_type_id(map), 0, "bpf_map__btf_value_type_id")) + goto out; + + skel->percpu_arr = bpf_map__initial_value(map, &data_sz); + if (!ASSERT_EQ(data_sz, desired_sz, "data_sz")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu_arr, "skel->percpu_arr")) + goto out; + if (!ASSERT_EQ(skel->percpu_arr->arr[0], arr_value, "arr[0]")) + goto out; + + err = test_global_percpu_data__load(skel); + if (bpf_map__value_size(map) > pcpu_max_value_size) { + if (ASSERT_EQ(err, -E2BIG, "test_global_percpu_data__load E2BIG")) + /* + * When the PAGE_SIZE is 64KiB, the new value size will + * be 128KiB. Creating a percpu_array map with the new + * value size will fail with -E2BIG due to percpu_array + * map value size limit 64KiB, which is calculated by + * PFN_ALIGN(32KiB). + * Let's skip the test for such case. + */ + test__skip(); + goto out; + } + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + map = skel->maps.percpu; + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + test_percpu_data_on_cpus(map, bpf_map__fd(map), prog_fd, &skel->bss->run); + ASSERT_EQ(skel->bss->arr_sum, arr_value * skel->bss->run, "arr_sum"); + +out: + test_global_percpu_data__destroy(skel); +} + static int create_rdonly_percpu_array(void) { LIBBPF_OPTS(bpf_map_create_opts, map_opts, @@ -388,6 +436,8 @@ void test_global_percpu_data(void) test_global_percpu_data_init(); if (test__start_subtest("lskel")) test_global_percpu_data_lskel(); + if (test__start_subtest("map_resize")) + test_global_percpu_data_map_resize(); if (test__start_subtest("rdonly_direct_read")) test_global_percpu_data_rdonly_direct_read(); if (test__start_subtest("rdonly_direct_write")) diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c index 5dc21b3b4cb5..2765cd46e9af 100644 --- a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -8,9 +8,12 @@ int loong SEC(".percpu.looooooooong"); int data3 SEC(".data.percpu"); int data2 SEC(".percpu.data"); +/* Used for testing bpf_map__set_value_size(). */ +int arr[1] SEC(".percpu.arr"); +int arr_sum; + int run; -/* cpu_id as array to verify map value resizing. */ -int cpu_id[1] SEC(".percpu"); +int cpu_id SEC(".percpu"); int data SEC(".percpu") = -1; int nums[7] SEC(".percpu"); bool set SEC(".percpu") = false; @@ -34,7 +37,8 @@ int update_percpu_data(void *ctx) data = 1; run++; set = true; - cpu_id[0] = bpf_get_smp_processor_id(); + cpu_id = bpf_get_smp_processor_id(); + arr_sum += arr[0]; return 0; } From cd35e1b10182c42b4ae31ee49119463b17d8ba7f Mon Sep 17 00:00:00 2001 From: Tw Date: Fri, 28 Aug 2026 08:34:10 +0800 Subject: [PATCH 42/90] libbpf: Fix for the potential undefined behavior due to shifting When compiling libbpf with sanitation, ubsan will report the following: ``` left shift of 1 by 31 places cannot be represented in type 'int' ``` This may lead to undefined behavior according to the compiler implementation, let's fix it by casting to unsigned counterpart before shifting. Signed-off-by: Tan Wei Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260828003448.1684064-1-tw19881113@gmail.com --- tools/lib/bpf/libbpf.c | 2 +- tools/lib/bpf/libbpf_internal.h | 4 ++-- tools/testing/selftests/bpf/test_btf.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index b749c01742ee..f09cbfd8e729 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -11738,7 +11738,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, errstr(bit)); return bit; } - attr.config |= 1 << bit; + attr.config |= 1ULL << bit; } attr.size = attr_sz; attr.type = type; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4c46d34fc055..cb4d96233844 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -77,7 +77,7 @@ #define JUMPTABLES_SEC ".jumptables" #define BTF_INFO_ENC(kind, kind_flag, vlen) \ - ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) + (((__u32)(!!(kind_flag)) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) #define BTF_INT_ENC(encoding, bits_offset, nr_bits) \ ((encoding) << 24 | (bits_offset) << 16 | (nr_bits)) @@ -259,7 +259,7 @@ static inline enum btf_func_linkage btf_func_linkage(const struct btf_type *t) static inline __u32 btf_type_info(int kind, int vlen, int kflag) { - return (kflag << 31) | (kind << 24) | vlen; + return ((__u32)kflag << 31) | (kind << 24) | vlen; } enum map_def_parts { diff --git a/tools/testing/selftests/bpf/test_btf.h b/tools/testing/selftests/bpf/test_btf.h index e65889ab4adf..e7bc78108374 100644 --- a/tools/testing/selftests/bpf/test_btf.h +++ b/tools/testing/selftests/bpf/test_btf.h @@ -7,7 +7,7 @@ #define BTF_END_RAW 0xdeadbeef #define BTF_INFO_ENC(kind, kind_flag, vlen) \ - ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) + (((__u32)(!!(kind_flag)) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) #define BTF_TYPE_ENC(name, info, size_or_type) \ (name), (info), (size_or_type) From f62a9b82700c718cdf619fb36aaab31015ff87eb Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:19 -0700 Subject: [PATCH 43/90] bpf: Record each half of a paired return value in verifier diagnostics A subprogram returning more than 8 bytes comes back in the R0:R2 register pair. There are three places, check_func_call(), prepare_func_exit() and check_kfunc_call(), where only R0 is tracked by the diagnostic modification scope and R2 is missed. Fix it by opening a diagnostic modification scope for each return register. This way, both return registers are recorded. Fixes: 0630ad00d96d ("bpf: Add verifier support for 16-byte returns in R0: R2") Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061519.1692270-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 56 ++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e036ae20bf6b..5d8162e13c20 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9995,9 +9995,14 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, */ if (!returns_void) { nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; - for (i = 0; i < nregs; i++) - mark_reg_unknown(env, caller->regs, ret_regs[i]); + mark_reg_unknown(env, caller->regs, ret_regs[0]); bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL, + BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, caller->regs, ret_regs[i]); + bpf_diag_mod_end(env); + } } if (env->subprog_info[subprog].might_throw) { @@ -10403,10 +10408,14 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) * return to the caller whatever the callee had in the * return register(s) */ - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - for (i = 0; i < nregs; i++) - caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; - bpf_diag_mod_end(env); + for (i = 0; i < nregs; i++) { + u32 regno = ret_regs[i]; + + bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], + BPF_DIAG_MOD_WRITE); + caller->regs[regno] = callee->regs[regno]; + bpf_diag_mod_end(env); + } } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -11342,15 +11351,33 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* * Mark the register(s) holding a @size byte kfunc return value as unknown - * scalars. Both halves of a register pair are treated the same way. + * scalars and return how many registers that took. Both halves of a register + * pair are treated the same way. + * + * R0 is written in the diagnostic modification scope the caller opened before + * the caller-saved scrub, since R0 gets no scrub record of its own and that + * scope is what carries its pre-call value into the history. A scope tracks a + * single register and does not nest, so close it before recording the rest of + * the return value. A value that fits in R0 leaves the caller's scope open for + * the caller to close, so a later write to R0 still lands in it. */ -static void mark_kfunc_ret_regs(struct bpf_verifier_env *env, - struct bpf_reg_state *regs, u32 size) +static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) { u32 i, nregs = ret_regs_cnt(size); - for (i = 0; i < nregs; i++) + mark_reg_unknown(env, regs, ret_regs[0]); + if (nregs == 1) + return nregs; + + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, ®s[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE); mark_reg_unknown(env, regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + + return nregs; } static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) @@ -13767,7 +13794,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id; + u32 i, nargs, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -14021,7 +14048,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_kfunc_ret_regs(env, regs, t->size); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); @@ -14039,7 +14066,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, btf_name_by_offset(desc_btf, t->name_off)); return -EINVAL; } - mark_kfunc_ret_regs(env, regs, t->size); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -14170,7 +14197,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * Record R0 before process_iter_next_call() snapshots the alternate * iterator path's diagnostic position. */ - bpf_diag_mod_end(env); + if (ret_nregs == 1) + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); From 77983371742e37ff3117acdc5dfb1969cedd415c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:24 -0700 Subject: [PATCH 44/90] bpf: Drop the recursion depth argument of btf_type_is_scalar_struct() btf_type_is_scalar_struct() recurses into nested struct members and carries the nesting depth in a @rec argument, so every caller has to spell out the 0 that starts the walk. Move the recursion into btf_struct_member_walk() and leave btf_type_is_scalar_struct() as a thin wrapper over it. This is groundwork for the later patches, which give the walk further state that callers should not have to supply: the kinds of member it accepts, and a record of the member that made it fail. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061524.1692957-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 2 +- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 24 +++++++++++++++--------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 004b06785521..3eb61edc8c5e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1489,7 +1489,7 @@ struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec); + const struct btf_type *t); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 91b8ce77f699..47d43eb983a5 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7995,7 +7995,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt */ bool local_func = subprog && !is_global; - if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + if (local_func || btf_type_is_scalar_struct(env, btf, t)) return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5d8162e13c20..ed077a5422af 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11647,9 +11647,8 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11667,7 +11666,7 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_struct_member_walk(env, btf, member_type, rec + 1)) return false; continue; } @@ -11686,6 +11685,13 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, return true; } +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_member_walk(env, btf, t, 0); +} + enum kfunc_ptr_arg_type { KF_ARG_CONST_MEM_SIZE, KF_ARG_MEM_SIZE, @@ -12066,7 +12072,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12082,7 +12088,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13138,7 +13144,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; } - if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13680,7 +13686,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14059,7 +14065,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_type_is_scalar_struct(env, desc_btf, t)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), From 3fc3e7331af6f60979fdf55e776bfe67105b584a Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:29 -0700 Subject: [PATCH 45/90] bpf: Add btf_type_is_arena_ptr() Add btf_type_is_arena_ptr() to test whether a BTF type is a pointer carrying the "arena" type tag, and use it to simplify both btf_scan_type_tags() and btf_validate_return_type(). The helper has external linkage because a later patch calls it from verifier.c to decide whether a struct member returned by value may be an arena pointer. btf_scan_type_tags() drove btf_type_tag_walk() with a one-entry match table, so the "multiple type tags" error it checked for could never fire: btf_type_tag_walk() only fails when the walk sets more than one flag. Dropping that error path, and with it the int return, is therefore no functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061529.1693266-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/btf.h | 1 + kernel/bpf/btf.c | 62 ++++++++++++++++----------------------------- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f117..ddd0f4f32d24 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -235,6 +235,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 47d43eb983a5..280530d25886 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3523,6 +3523,22 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena")) + return true; + } + + return false; +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -7927,51 +7943,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; - - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -7979,7 +7963,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ @@ -8106,9 +8090,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) From 8d7ea1b752e5e49b00442f27f2a76228423aa74b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:34 -0700 Subject: [PATCH 46/90] bpf: Let the by-value struct walk take the kinds of member it accepts btf_struct_member_walk() hardcodes that every member of a struct returned by value must be a scalar. A later patch needs the same walk to also accept arena pointers, so give it a mask of the member kinds it allows. The mask reaches callers through btf_struct_is_composed_of(), which replaces btf_type_is_scalar_struct() as the entry point exported to btf.c. btf_type_is_scalar_struct() stays as a verifier-local wrapper for the call sites that only ever ask about scalars. There is no functional change: btf_member_kind_allowed() is exactly btf_type_is_scalar() when the mask is BTF_MEMBER_SCALAR alone, and the shared check is the last statement of the loop body, so dropping the array branch's continue neither skips a member nor checks one twice. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061534.1693789-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 11 +++++++++-- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 38 ++++++++++++++++++++++++++---------- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 3eb61edc8c5e..ae9f606539f4 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,8 +1488,15 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t); + +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 280530d25886..b1f4ef614d4c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7979,7 +7979,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt */ bool local_func = subprog && !is_global; - if (local_func || btf_type_is_scalar_struct(env, btf, t)) + if (local_func || btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ed077a5422af..90139f1b78d1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11646,9 +11646,23 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, 4 levels of nesting allowed. An array member counts as its + * element type. + */ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec) + const struct btf_type *t, u32 member_kinds, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11666,7 +11680,7 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!btf_struct_member_walk(env, btf, member_type, rec + 1)) + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, rec + 1)) return false; continue; } @@ -11675,21 +11689,25 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt if (!array->nelems) return false; member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; - continue; } - if (!btf_type_is_scalar(member_type)) + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) return false; } return true; } -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t) + const struct btf_type *t, u32 member_kinds) { - return btf_struct_member_walk(env, btf, t, 0); + return btf_struct_member_walk(env, btf, t, member_kinds, 0); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); } enum kfunc_ptr_arg_type { From 9e0195cd1d312ed99c3107d82f418e97baa45289 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:39 -0700 Subject: [PATCH 47/90] bpf: Let a by-value struct nest arrays and structs freely Go through nested array types until it reaches a non-array type. Later code will check such non-array type based on current logic. A zero-length array is still rejected, at whichever dimension it appears. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061539.1695015-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 90139f1b78d1..a377b1995b51 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11675,6 +11675,17 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt const struct btf_array *array; member_type = btf_type_skip_modifiers(btf, member->type, NULL); + /* + * Every element of an array is laid out in the value being + * returned, so an array counts as its element type however many + * dimensions deep that is. + */ + while (btf_type_is_array(member_type)) { + array = btf_array(member_type); + if (!array->nelems) + return false; + member_type = btf_type_skip_modifiers(btf, array->type, NULL); + } if (btf_type_is_struct(member_type)) { if (rec >= 3) { verbose(env, "max struct nesting depth exceeded\n"); @@ -11684,12 +11695,6 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt return false; continue; } - if (btf_type_is_array(member_type)) { - array = btf_array(member_type); - if (!array->nelems) - return false; - member_type = btf_type_skip_modifiers(btf, array->type, NULL); - } if (!btf_member_kind_allowed(btf, member_type, member_kinds)) return false; } From 282d07dc6c74a5b4eea19743dd8c487c4b8814da Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:44 -0700 Subject: [PATCH 48/90] bpf: Report which member makes a kfunc return type unsupported A kfunc that returns a struct by value may only return scalars, and the message that rejects one names the type but not the member at fault: kernel function bpf_kfunc_call_test_ret_ptr returns STRUCT prog_test_ret_ptr that is not composed of scalars Record the member that made the walk fail and name it. What is recorded is a path, not a single member, because the walk descends up to 4 levels. For struct outer { struct inner { void *p; } in; __u64 tag; }; naming 'p' alone would send the reader looking for a member struct outer does not have, so the actual message reads "member 'in.p' has type PTR". For struct s { void *p[2]; }; naming 'p' alone would blame the member, when it is the element type that was rejected, so the actual message reads "member 'p[]' has type PTR". A zero-length array has no element type to name and is rejected for its length instead, so it reports "member 'p' is a zero-length array". The detailed diagnostics for this failure: Verification failed: Program Structure: Unsupported kernel function return type Reason: bpf_kfunc_call_test_ret_ptr() returns STRUCT prog_test_ret_ptr by value. Its member 'p' is PTR, not a scalar. Only kfuncs returning scalar values, or structures composed of scalar values are supported. ... Suggestion: Call a kernel function that returns only scalars by value. A type nested deeper than the walk descends has no single member to blame, so that case reports the depth instead: Reason: bpf_kfunc_call_test_ret_deep() returns STRUCT prog_test_ret_deep by value. It nests structs more than 4 levels deep. ... Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061545.1695736-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 103 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 9 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a377b1995b51..745daf58b052 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11646,6 +11646,17 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } +#define BTF_MEMBER_MAX_DEPTH 4 +#define BTF_MEMBER_PATH_LEN 64 + +struct btf_member_path { + const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; + bool in_array[BTF_MEMBER_MAX_DEPTH]; + const struct btf_type *bad_type; + int depth; + bool too_deep; +}; + static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, u32 member_kinds) { @@ -11662,10 +11673,12 @@ static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type * element type. */ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, u32 member_kinds, int rec) + const struct btf_type *t, u32 member_kinds, int rec, + struct btf_member_path *path) { const struct btf_type *member_type; const struct btf_member *member; + bool in_array; u32 i; if (!btf_type_is_struct(t)) @@ -11674,6 +11687,7 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt for_each_member(i, t, member) { const struct btf_array *array; + in_array = false; member_type = btf_type_skip_modifiers(btf, member->type, NULL); /* * Every element of an array is laid out in the value being @@ -11682,30 +11696,48 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt */ while (btf_type_is_array(member_type)) { array = btf_array(member_type); - if (!array->nelems) - return false; + if (!array->nelems) { + member_type = NULL; + goto bad_member; + } member_type = btf_type_skip_modifiers(btf, array->type, NULL); + in_array = true; } if (btf_type_is_struct(member_type)) { - if (rec >= 3) { + if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { verbose(env, "max struct nesting depth exceeded\n"); + if (path) + path->too_deep = true; return false; } - if (!btf_struct_member_walk(env, btf, member_type, member_kinds, rec + 1)) - return false; + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, + rec + 1, path)) + goto bad_path; continue; } if (!btf_member_kind_allowed(btf, member_type, member_kinds)) - return false; + goto bad_member; } return true; + +bad_member: + if (path) { + path->depth = rec + 1; + path->bad_type = member_type; + } +bad_path: + if (path && path->depth) { + path->member[rec] = member; + path->in_array[rec] = in_array; + } + return false; } bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, const struct btf_type *t, u32 member_kinds) { - return btf_struct_member_walk(env, btf, t, member_kinds, 0); + return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); } static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, @@ -11715,6 +11747,25 @@ static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); } +static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, + char *buf, size_t buf_sz) +{ + size_t len = 0; + int i; + + buf[0] = '\0'; + for (i = 0; i < path->depth; i++) { + const char *name = btf_name_by_offset(btf, path->member[i]->name_off); + + /* An anonymous struct or union has no name to spell. */ + if (!name || !name[0]) + continue; + len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name); + if (path->in_array[i]) + len += scnprintf(buf + len, buf_sz - len, "[]"); + } +} + enum kfunc_ptr_arg_type { KF_ARG_CONST_MEM_SIZE, KF_ARG_MEM_SIZE, @@ -14082,17 +14133,51 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); } else if (btf_type_is_struct(t)) { + struct btf_member_path path = {}; + const char *member_note = ""; + /* * The returned struct comes back as raw register bits modeled * as an unknown scalar, so it must contain only scalars: * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t)) { + if (!btf_struct_member_walk(env, desc_btf, t, BTF_MEMBER_SCALAR, 0, &path)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off)); + if (path.too_deep) { + member_note = bpf_diag_fmt( + env, " It nests structs more than %d levels deep.", + BTF_MEMBER_MAX_DEPTH); + } else if (path.depth) { + char bad_name[BTF_MEMBER_PATH_LEN]; + + btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); + if (!path.bad_type) { + verbose(env, "member '%s' is a zero-length array\n", + bad_name); + member_note = bpf_diag_fmt( + env, " Its member '%s' is a zero-length array.", + bad_name); + } else { + verbose(env, "member '%s' has type %s\n", bad_name, + btf_type_str(path.bad_type)); + member_note = bpf_diag_fmt( + env, " Its member '%s' is %s, not a scalar.", + bad_name, btf_type_str(path.bad_type)); + } + } + bpf_diag_program_structure( + env, insn_idx, "unsupported kernel function return type", + "Call a kernel function that returns only scalars by value.", + "%s() returns %s %s by value.%s " + "Only kfuncs returning scalar values, or " + "structures composed of scalar values are " + "supported.", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off), member_note); return -EINVAL; } ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); From ff08e6b79ee1470e736054e9558c47e4150406b1 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:50 -0700 Subject: [PATCH 49/90] bpf: Allow a global function to return arena pointers by value A global function may already return an arena pointer on its own, and check_global_ret_scalar_reg() accepts one in either half of the R0:R2 pair. Let the members of a by-value struct it returns be arena pointers as well, rather than scalars only. Only a subprogram gets this: the main program returns to the kernel, which has no arena to cast the address back into, so btf_validate_return_type() keeps rejecting an arena pointer there, whether bare or inside a struct. The rejection message therefore has to name what the function at hand may return, or it would offer the main program an arena pointer it cannot have. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061550.1696869-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 28 +++++++++++++++---- .../selftests/bpf/progs/exceptions_fail.c | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index b1f4ef614d4c..9c2cab08bb79 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7973,13 +7973,23 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_struct(t) && t->size <= 16) { /* * A global function's caller models the return as an opaque - * scalar pair, so it may only return scalars by value. A local - * function is verified inline, so a pointer field stays tracked - * and needs no such restriction. + * scalar pair, so a pointer member would be laundered into a + * scalar and escape provenance and reference tracking. Only + * scalars and arena pointers are allowed: an arena pointer has + * no provenance to lose, since a program may already derive one + * from any scalar with addr_space_cast(), which confines the + * result to the arena. The main program is the exception: it + * returns to the kernel, which has no arena to cast the address + * back into. A local function is verified inline, so its caller + * receives the real register state and any member is fine. */ bool local_func = subprog && !is_global; + u32 member_kinds = BTF_MEMBER_SCALAR; - if (local_func || btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) + if (subprog) + member_kinds |= BTF_MEMBER_ARENA_PTR; + + if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) return 0; } @@ -8073,10 +8083,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { + /* Only a subprogram may return arena pointers. */ + const char *supported = subprog ? + "void, scalar, arena pointer, or a struct/union of " + "scalars and arena pointers" : + "void, scalar, or a scalar-only struct/union"; + bpf_log(log, "Global function %s() has unsupported return type. " - "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", - tname); + "Only %s up to 16 bytes is supported.\n", + tname, supported); } return err; } diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 9708efb93683..22503cf62e9f 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") +__failure __msg("Only void, scalar, arena pointer, or a struct/union of scalars and arena pointers up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); From 9046109974dd1f3ffc3f62e22555be13eaf7fdb0 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:55 -0700 Subject: [PATCH 50/90] bpf: Allow arena pointers in a by-value kfunc return A kfunc may already return an arena pointer on its own, which the program casts back into the arena address space to use. Let the members of a by-value struct it returns be arena pointers as well, rather than scalars only. Such a member carries the arena type tag but not the address space qualifier, so the program has to cast it itself. Handing it back as a scalar costs nothing: an arena address has no provenance to track, and the cast is confined to the caller's arena. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061555.1697503-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 22 +++++++++++-------- .../selftests/bpf/progs/aggregate_ret_kfunc.c | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 745daf58b052..a725e3917976 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14138,13 +14138,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* * The returned struct comes back as raw register bits modeled - * as an unknown scalar, so it must contain only scalars: - * otherwise a pointer field would be laundered into a scalar - * and escape provenance and reference tracking. + * as an unknown scalar, so a pointer member would be laundered + * into a scalar and escape provenance and reference tracking. + * Only scalars and arena pointers are allowed; see + * btf_validate_return_type() in btf.c for why an arena pointer + * is safe here. */ - if (!btf_struct_member_walk(env, desc_btf, t, BTF_MEMBER_SCALAR, 0, &path)) { + if (!btf_struct_member_walk(env, desc_btf, t, + BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { verbose(env, - "kernel function %s returns %s %s that is not composed of scalars\n", + "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off)); if (path.too_deep) { @@ -14165,16 +14168,17 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "member '%s' has type %s\n", bad_name, btf_type_str(path.bad_type)); member_note = bpf_diag_fmt( - env, " Its member '%s' is %s, not a scalar.", + env, + " Its member '%s' is %s, not a scalar or an arena pointer.", bad_name, btf_type_str(path.bad_type)); } } bpf_diag_program_structure( env, insn_idx, "unsupported kernel function return type", - "Call a kernel function that returns only scalars by value.", + "Call a kernel function that returns only scalars or arena pointers by value.", "%s() returns %s %s by value.%s " - "Only kfuncs returning scalar values, or " - "structures composed of scalar values are " + "Only kfuncs returning scalar values or arena pointers, or " + "structures composed of scalar values or arena pointers are " "supported.", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off), member_note); diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c index d6b422ae9784..f10e5cf6fd89 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -71,7 +71,7 @@ __naked int aggregate_ret_kfunc_fastcall_fail(void) SEC("tc") __arch_x86_64 __arch_arm64 -__failure __msg("is not composed of scalars") +__failure __msg("is not composed of scalars or arena pointers") __naked int aggregate_ret_kfunc_ptr_fail(void) { asm volatile ( From cc03706d92c9948bc99702599872719e3dc29add Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:16:00 -0700 Subject: [PATCH 51/90] selftests/bpf: Check the member named for an unsupported kfunc return type Add tests to cover cases where a kfunc return type is rejected with proper messages including member names and array types. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061600.1697800-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/aggregate_ret_kfunc.c | 82 +++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 40 +++++++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 42 ++++++++++ 3 files changed, 164 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c index f10e5cf6fd89..0361a7b42c9b 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -19,6 +19,11 @@ void __kfunc_btf_root(void) "r"(&bpf_kfunc_call_test_ret_fastcall), "r"(&bpf_kfunc_call_test_ret_ptr), "r"(&bpf_kfunc_call_test_ret_ii), + "r"(&bpf_kfunc_call_test_ret_nested), + "r"(&bpf_kfunc_call_test_ret_ptr_arr), + "r"(&bpf_kfunc_call_test_ret_deep), + "r"(&bpf_kfunc_call_test_ret_arr_struct), + "r"(&bpf_kfunc_call_test_ret_arr2d), "r"(&bpf_kfunc_call_test_ret_big)); } @@ -72,6 +77,7 @@ __naked int aggregate_ret_kfunc_fastcall_fail(void) SEC("tc") __arch_x86_64 __arch_arm64 __failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p' has type PTR") __naked int aggregate_ret_kfunc_ptr_fail(void) { asm volatile ( @@ -84,6 +90,82 @@ __naked int aggregate_ret_kfunc_ptr_fail(void) : __clobber_all); } +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in.p' has type PTR") +__naked int aggregate_ret_kfunc_nested_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_nested];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_nested) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p[]' has type PTR") +__naked int aggregate_ret_kfunc_ptr_arr_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_ptr_arr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr_arr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in1[].in2.p' has type PTR") +__naked int aggregate_ret_kfunc_arr_struct_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr_struct];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr_struct) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'a[].p' has type PTR") +__naked int aggregate_ret_kfunc_arr2d_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arr2d];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arr2d) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("max struct nesting depth exceeded") +__naked int aggregate_ret_kfunc_too_deep_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_deep];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_deep) + : __clobber_all); +} + SEC("tc") __arch_x86_64 __arch_arm64 __failure __msg("R2 !read_ok") diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 850cf4f830c4..dff1ad6f491f 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -981,6 +981,41 @@ __bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) return r; } +__bpf_kfunc struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(u64 tag) +{ + struct prog_test_ret_nested r = { .in = { .p = NULL }, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) +{ + struct prog_test_ret_ptr_arr r = { .p = { NULL, NULL } }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) +{ + struct prog_test_ret_arr_struct r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) +{ + struct prog_test_ret_arr2d r = {}; + + return r; +} + +__bpf_kfunc struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(u64 v) +{ + struct prog_test_ret_deep r = { .l1 = { .l2 = { .l3 = { .l4 = { .v = v } } } } }; + + return r; +} + __bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) { struct prog_test_ret_ii r = { .a = a, .b = b }; @@ -1539,6 +1574,11 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_nested) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr_arr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr2d) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) #endif BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 65e693ada736..9202c31ba6c9 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -70,6 +70,43 @@ struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ __u64 tag; }; +struct prog_test_ret_nested { /* 16 bytes: the pointer hides one level down */ + struct { + void *p; + } in; + __u64 tag; +}; + +struct prog_test_ret_ptr_arr { /* 16 bytes: an array of pointers */ + void *p[2]; +}; + +struct prog_test_ret_arr_struct { /* 16 bytes: the pointer is under an array of structs */ + struct { + struct { + void *p; + } in2; + } in1[2]; +}; + +struct prog_test_ret_arr2d { /* 16 bytes: a two dimensional array of structs */ + struct { + void *p; + } a[1][2]; +}; + +struct prog_test_ret_deep { /* 8 bytes, but nested past the 4-level walk limit */ + struct { + struct { + struct { + struct { + __u64 v; + } l4; + } l3; + } l2; + } l1; +}; + struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ __u64 a; __u64 b; @@ -159,6 +196,11 @@ struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; +struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; +struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) __ksym; +struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) __ksym; +struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(__u64 v) __ksym; struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, From 18bc6183884279b279026737b1fa7cc0517010bc Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:16:05 -0700 Subject: [PATCH 52/90] selftests/bpf: Test global functions returning arena pointers by value Cover the by-value struct returns a global function may now make: two arena pointers filling R0:R2, an arena pointer beside a scalar, an array of them, and an eight byte struct returned in R0 alone. The existing cases for a struct and a union carrying a plain pointer stay rejected. check_arena_struct_ret() also stores through both halves of the returned pair and reads them back, so the test covers the returned pointers still being usable as arena pointers rather than only the program verifying. A toolchain guard goes into each file, and they differ because the constructs do. aggregate_ret_func.c uses "#if defined(__clang__)". Those cases are __naked, so the function body is the inline asm and nothing else: the clang compiler never lowers the return, and the 16 byte return type reaches the verifier only through BTF. The minimum checked is LLVM 21. gcc is excluded because it returns a by-value struct through a hidden pointer and emits the 'r0 = r1' returning it after the __naked body's exit, leaving the subprogram falling through. verifier_arena.c uses "#if defined(__clang_major__) && __clang_major__ >= 23". arena_word_pair() returns the struct from C, so the compiler lowers the return itself, and a 16 byte return only lands in R0:R2 with the LLVM 23 BPF ABI. An older clang does not fall back to anything here, it rejects the function with "aggregate returns are not supported", so the floor is what lets the file build at all. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061605.1698601-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/progs/aggregate_ret_func.c | 146 ++++++++++++++++++ .../selftests/bpf/progs/verifier_arena.c | 48 ++++++ 2 files changed, 194 insertions(+) diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c index 6f66fc822ced..f0077d951a25 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include #include +#include #include "bpf_misc.h" typedef unsigned __int128 u128; @@ -234,4 +235,149 @@ __naked int aggregate_ret_global_union_ptr_fail(void) #endif +/* + * gcc returns a by-value struct through a hidden pointer, and emits the + * 'r0 = r1' returning it after the __naked body's exit, leaving the + * subprogram falling through. Build these with clang only. + */ +#if defined(__clang__) + +struct arena_pair { + void __arena *lo; + void __arena *hi; +}; + +struct arena_and_scalar { + void __arena *p; + __u64 x; +}; + +struct arena_array { + void __arena *p[2]; +}; + +struct arena_single { + void __arena *p; +}; + +union arena_upair { + void __arena *p; + __u64 halves[2]; +}; + +__naked struct arena_pair global_ret_arena_pair(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_pair(void) +{ + asm volatile ( + "call %[global_ret_arena_pair];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_pair) + : __clobber_all); +} + +__naked struct arena_and_scalar global_ret_arena_and_scalar(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_and_scalar(void) +{ + asm volatile ( + "call %[global_ret_arena_and_scalar];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_and_scalar) + : __clobber_all); +} + +__naked struct arena_array global_ret_arena_array(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_array(void) +{ + asm volatile ( + "call %[global_ret_arena_array];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_array) + : __clobber_all); +} + +__naked struct arena_single global_ret_arena_single(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__success __retval(0) +__naked int aggregate_ret_global_arena_single(void) +{ + asm volatile ( + "call %[global_ret_arena_single];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_single) + : __clobber_all); +} + +__naked union arena_upair global_ret_arena_union(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_union(void) +{ + asm volatile ( + "call %[global_ret_arena_union];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_union) + : __clobber_all); +} + +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 815f342eb4b0..332322c9b54c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -734,4 +734,52 @@ int check_arena_arg_ret(void *ctx) return 0; } +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct arena_word_pair { + u32 __arena *first; + u32 __arena *second; +}; + +__weak struct arena_word_pair arena_word_pair(u32 __arena *page) +{ + struct arena_word_pair pair; + + pair.first = page; + pair.second = page + 1; + + return pair; +} + +SEC("syscall") +__load_if_JITed() +__success __retval(0) +int check_arena_struct_ret(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *first, *second; + struct arena_word_pair pair; + + if (!page) + return 1; + + pair = arena_word_pair(page); + if (!pair.first || !pair.second) + return 2; + + /* Both halves of the pair must still be usable as arena pointers. */ + first = pair.first; + second = pair.second; + *first = 1; + *second = 2; + if (*first != 1) + return 3; + if (*second != 2) + return 4; + + return 0; +} + +#endif + char _license[] SEC("license") = "GPL"; From f7213961e8e24b3ed98b13406c06fcf91138ad2f Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:16:10 -0700 Subject: [PATCH 53/90] selftests/bpf: Test kfuncs returning arena pointers by value Cover the by-value struct returns a kfunc may now make: two arena pointers filling R0:R2, and an arena pointer beside a scalar. Two further cases drop the tag from one member of a struct and one arm of a union, and stay rejected naming that member, so what decides is the tag rather than the member being a pointer. The existing cases for a struct and a nested struct carrying a plain pointer stay rejected as well. These cases call the kfuncs from C, so the compiler lowers the by-value return itself, and a struct or union only lands in R0:R2 with the LLVM 23 BPF ABI. An older clang, and gcc, return it through a hidden pointer in R1 instead, which shifts the arguments along and fails verification. The file is therefore guarded on LLVM 23, falling back to a dummy test. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061610.1699950-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/aggregate_ret.c | 42 ++++++ .../bpf/progs/aggregate_ret_kfunc_arena.c | 121 ++++++++++++++++++ .../selftests/bpf/test_kmods/bpf_testmod.c | 32 +++++ .../bpf/test_kmods/bpf_testmod_kfunc.h | 30 +++++ 4 files changed, 225 insertions(+) create mode 100644 tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c index e0b94ed10f94..07d9d6e1d6b8 100644 --- a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c @@ -1,11 +1,53 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include +#include #include "aggregate_ret_func.skel.h" #include "aggregate_ret_kfunc.skel.h" +#include "aggregate_ret_kfunc_arena.skel.h" + +static bool testmod_has_arena_tagged_member(void) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_type *t; + bool tagged = false; + __s32 id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return false; + + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) + goto out; + + /* prog_test_ret_arena::a is 'void __arena_tag *': PTR -> TYPE_TAG -> void */ + id = btf__find_by_name_kind(module_btf, "prog_test_ret_arena", BTF_KIND_STRUCT); + if (id <= 0) + goto out; + + t = btf__type_by_id(module_btf, btf_members(btf__type_by_id(module_btf, id))[0].type); + if (!t || !btf_is_ptr(t)) + goto out; + + t = btf__type_by_id(module_btf, t->type); + tagged = t && btf_is_type_tag(t) && + !strcmp(btf__name_by_offset(module_btf, t->name_off), "arena"); + +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + + return tagged; +} void test_aggregate_ret(void) { RUN_TESTS(aggregate_ret_func); RUN_TESTS(aggregate_ret_kfunc); + + if (testmod_has_arena_tagged_member()) + RUN_TESTS(aggregate_ret_kfunc_arena); + else + test__skip(); } diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c new file mode 100644 index 000000000000..a0f263694a63 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 2); +} arena SEC(".maps"); + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena(void *ctx) +{ + u32 volatile __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + u32 volatile __arena *a, *b; + struct prog_test_ret_arena r; + + if (!page) + return 1; + + /* Both halves come back in R0:R2, pointing at page and page + 4. */ + r = bpf_kfunc_call_test_ret_arena((u64)page); + if (!r.a || !r.b) + return 2; + + a = (u32 __arena *)r.a; + b = (u32 __arena *)r.b; + *a = 1; + *b = 2; + if (*a != 1) + return 3; + if (*b != 2) + return 4; + + page[0] = 7; + if (*a != 7) + return 5; + page[1] = 9; + if (*b != 9) + return 6; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_arena_mixed(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + struct prog_test_ret_arena_mixed r; + u32 volatile __arena *p; + + if (!page) + return 1; + + /* An arena pointer in R0 beside a scalar in R2. */ + r = bpf_kfunc_call_test_ret_arena_mixed((u64)page); + if (!r.p) + return 2; + if (r.tag != 0xbeef) + return 3; + + p = (u32 __arena *)r.p; + *p = 3; + if (*p != 3) + return 4; + + return 0; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_untagged_fail(void *ctx) +{ + struct prog_test_ret_arena_untagged r; + + r = bpf_kfunc_call_test_ret_arena_untagged(0); + + return r.a == r.b; +} + +SEC("syscall") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'b' has type PTR") +int aggregate_ret_kfunc_arena_union_fail(void *ctx) +{ + union prog_test_ret_arena_union r; + + r = bpf_kfunc_call_test_ret_arena_union(0); + + return r.a == r.b; +} + +#else + +SEC("socket") +__description("aggregate_ret_kfunc_arena: needs LLVM 23, dummy test") +__skip("needs LLVM 23") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index dff1ad6f491f..2380b6cbdead 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -981,6 +981,34 @@ __bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) return r; } +__bpf_kfunc struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(u64 addr) +{ + struct prog_test_ret_arena r = { .a = (void *)addr, .b = (void *)(addr + 4) }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(u64 addr) +{ + struct prog_test_ret_arena_mixed r = { .p = (void *)addr, .tag = 0xbeef }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(u64 addr) +{ + struct prog_test_ret_arena_untagged r = { .a = (void *)addr, .b = NULL }; + + return r; +} + +__bpf_kfunc union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(u64 addr) +{ + union prog_test_ret_arena_union r = { .a = (void *)addr }; + + return r; +} + __bpf_kfunc struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(u64 tag) { struct prog_test_ret_nested r = { .in = { .p = NULL }, .tag = tag }; @@ -1574,6 +1602,10 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_mixed) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_untagged) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_union) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_nested) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr_arr) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arr_struct) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 9202c31ba6c9..b213ef14848b 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -26,6 +26,12 @@ struct prog_test_ref_kfunc { }; #endif +#if __has_attribute(btf_type_tag) +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#else +#define __arena_tag +#endif + struct bpf_iter_testmod_seq; struct prog_test_pass1 { @@ -70,6 +76,26 @@ struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ __u64 tag; }; +struct prog_test_ret_arena { /* 16 bytes: two arena pointers */ + void __arena_tag *a; + void __arena_tag *b; +}; + +struct prog_test_ret_arena_mixed { /* 16 bytes: an arena pointer and a scalar */ + void __arena_tag *p; + __u64 tag; +}; + +struct prog_test_ret_arena_untagged { /* 16 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + +union prog_test_ret_arena_union { /* 8 bytes: 'b' lacks the arena tag */ + void __arena_tag *a; + void *b; +}; + struct prog_test_ret_nested { /* 16 bytes: the pointer hides one level down */ struct { void *p; @@ -200,6 +226,10 @@ struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; struct prog_test_ret_ptr_arr bpf_kfunc_call_test_ret_ptr_arr(void) __ksym; struct prog_test_ret_arr_struct bpf_kfunc_call_test_ret_arr_struct(void) __ksym; struct prog_test_ret_arr2d bpf_kfunc_call_test_ret_arr2d(void) __ksym; +struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(__u64 addr) __ksym; +struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(__u64 addr) __ksym; +struct prog_test_ret_arena_untagged bpf_kfunc_call_test_ret_arena_untagged(__u64 addr) __ksym; +union prog_test_ret_arena_union bpf_kfunc_call_test_ret_arena_union(__u64 addr) __ksym; struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(__u64 v) __ksym; struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, From 25b0d73a7742d7415156c607f6278682efc12fc2 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:16:15 -0700 Subject: [PATCH 54/90] docs/bpf: Document arena pointers in a by-value return Section 2.9 describes the by-value return contract as scalars only, which no longer holds: a kfunc and a global subprogram may now return a struct or union whose members are scalars or arena pointers. Update it. Note that an arena pointer member is handed back as a scalar like every other member. That costs nothing: the program has to cast_kern() the value before it can be used, and the verifier allows that cast on any scalar, so the member gives the program no reach it did not already have. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061615.1700421-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- Documentation/bpf/kfuncs.rst | 39 ++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 89dea6b0b024..ebe37c1fe9d2 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -581,20 +581,29 @@ against the arena. Larger accesses must verify the range explicitly. A kfunc may return a scalar, a pointer, or a small struct or union by value. A scalar or pointer of up to 8 bytes is returned in R0, as usual. -A struct or union returned by value must be composed only of scalars -(recursively), where a scalar is an integer or an enum; arrays of scalars are -allowed as members. Its bytes are handed back to the program as the raw -contents of R0 (and R2), so a pointer field would be laundered into a scalar -and escape the verifier's pointer provenance and reference tracking. A struct -or union with a pointer member is therefore rejected at load time, and so is -one with a floating-point member, which the ABI may not return in R0:R2 at +A struct or union returned by value must be composed only of scalars and arena +pointers, where a scalar is an integer or an enum and an arena pointer is one +carrying the ``btf_type_tag("arena")`` attribute. Those may be nested in +structs and unions and in arrays of any number of dimensions, in any +combination, as long as what the nesting bottoms out in is a scalar or an arena +pointer. Its bytes are handed back to the program as the raw contents of R0 +(and R2), so a member of any other pointer type would be laundered into a +scalar and escape the verifier's pointer provenance and reference tracking. A +struct or union with such a member is therefore rejected at load time, and so +is one with a floating-point member, which the ABI may not return in R0:R2 at all. +An arena pointer member is handed back as a scalar too, but nothing is lost by +that. The program must ``cast_kern()`` the value before it can be used, and the +verifier allows that cast on any scalar, so a laundered arena address gives the +program no reach it did not already have. The result is confined to the +program's arena in either case. + A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a -scalar-only struct or union, or an ``__int128``. Such a value is returned -in the register pair R0:R2, matching the convention LLVM uses for the BPF -target: the first 8 bytes in R0 and the second 8 bytes in R2. A struct or -union of 8 bytes or less is returned in R0 alone. +struct or union of scalars and arena pointers, or an ``__int128``. Such a +value is returned in the register pair R0:R2, matching the convention LLVM +uses for the BPF target: the first 8 bytes in R0 and the second 8 bytes in R2. +A struct or union of 8 bytes or less is returned in R0 alone. :: @@ -620,10 +629,10 @@ used when the program is JITed, since the interpreter propagates only R0 out of a subprogram: without a JIT the return value stays in R0 alone, and a caller reading R2 is rejected for reading an uninitialized register. A global subprogram is verified in isolation, so its by-value struct or union return is -restricted to scalars just like a kfunc's; a static subprogram is verified -inline and has no such restriction. The main program is not covered: its return -value is the program's exit code, read out of R0 alone, so a declared upper -half is never looked at. +restricted to scalars and arena pointers just like a kfunc's; a static +subprogram is verified inline and has no such restriction. The main program is +not covered: its return value is the program's exit code, read out of R0 +alone, so a declared upper half is never looked at. A global subprogram must leave a scalar in *every* register of the pair, so both halves of the returned value have to be assigned. Leaving the upper half From 264d8fd2794fabe76a89abd5d4f5f8c1ed5fad85 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:17 +0200 Subject: [PATCH 55/90] bpf, keys: Add a bpf keyring for program signature validation BPF program signatures can currently be verified against one of the system keyrings (builtin, secondary, platform) or against an arbitrary user/session caller-supplied keyring named through keyring_id. There is nothing in between: the system keyrings need a kernel rebuild or a vouched-for enrollment to rotate a key, while a caller-supplied keyring is fully controlled by the loader and therefore carries no trust on its own (unless explicitly combined with BPF LSM to protect against key tampering). Add a dedicated bpf keyring to fill that gap, modelled after the dm-verity keyring which was added in commit 033724b1c627 ("dm-verity: add dm-verity keyring") and which can eventually be used also via systemd [0] through the same enrollment method as in dm-verity's case. It is selected with the new KEY_SPEC_BPF_KEYRING special key id and gives an operator a place to enroll a BPF-only signing key at boot, specifically scoped to BPF program loading and nothing else in the kernel's trust hierarchy. The id is reserved from the KEY_SPEC space so that the latter is not linked into any process keyring, and lookup_user_key() resolves KEY_SPEC_BPF_KEYRING constant instead of having to look it up via /proc/keys first. By default the keyring is sealed empty at init. Systems that want to provision keys pass bpf.keyring_unsealed=1, which leaves the keyring open for the initrd to add keys to. The keyring is only ever consulted once it is both non-empty and restricted. An unrestricted keyring is ignored. Signed-off-by: Daniel Borkmann Link: https://github.com/systemd/systemd/pull/43549 [0] Link: https://lore.kernel.org/r/20260828175227.1537793-2-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- .../admin-guide/kernel-parameters.txt | 8 +++ include/linux/bpf.h | 7 ++ include/linux/key.h | 2 + include/uapi/linux/keyctl.h | 1 + kernel/bpf/Makefile | 3 + kernel/bpf/keys.c | 68 +++++++++++++++++++ kernel/bpf/verifier.c | 9 +++ security/keys/process_keys.c | 25 +++++++ 8 files changed, 123 insertions(+) create mode 100644 kernel/bpf/keys.c diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index e4643634a9b1..2beb61092bb3 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -667,6 +667,14 @@ Kernel parameters See Documentation/admin-guide/bootconfig.rst + bpf.keyring_unsealed= + [BPF] When set to 1, leave the bpf keyring unsealed + after initialization so that userspace can provision + keys. Once the keyring is restricted it becomes active + and can be used for BPF program signature verification. + + See Documentation/bpf/signing.rst + bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) bttv.radio= Most important insmod options are available as kernel args too. diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c2027688be3e..039674b2f875 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1735,6 +1735,7 @@ enum bpf_sig_keyring { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; struct bpf_prog_aux { @@ -3819,6 +3820,7 @@ struct bpf_key { #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); +struct bpf_key *bpf_lookup_keyring(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3839,6 +3841,11 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id) return NULL; } +static inline struct bpf_key *bpf_lookup_keyring(void) +{ + return NULL; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } diff --git a/include/linux/key.h b/include/linux/key.h index 81b8f05c6898..bd10fe45819d 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -440,6 +440,8 @@ extern key_ref_t keyring_search(key_ref_t keyring, extern int keyring_restrict(key_ref_t keyring, const char *type, const char *restriction); +extern void key_register_bpf_keyring(struct key *keyring); + extern struct key *key_lookup(key_serial_t id); static inline key_serial_t key_serial(const struct key *key) diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h index 4c8884eea808..fa85b9760391 100644 --- a/include/uapi/linux/keyctl.h +++ b/include/uapi/linux/keyctl.h @@ -24,6 +24,7 @@ #define KEY_SPEC_GROUP_KEYRING -6 /* - key ID for GID-specific keyring */ #define KEY_SPEC_REQKEY_AUTH_KEY -7 /* - key ID for assumed request_key auth key */ #define KEY_SPEC_REQUESTOR_KEYRING -8 /* - key ID for request_key() dest keyring */ +#define KEY_SPEC_BPF_KEYRING -9 /* - key ID for the BPF-specific keyring */ /* request-key default keyrings */ #define KEY_REQKEY_DEFL_NO_CHANGE -1 diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 90255d80e5be..9a92c348bbda 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += offload.o obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o obj-$(CONFIG_BPF_SYSCALL) += tcx.o endif +ifeq ($(CONFIG_KEYS),y) +obj-$(CONFIG_BPF_SYSCALL) += keys.o +endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c new file mode 100644 index 000000000000..08549b3220c1 --- /dev/null +++ b/kernel/bpf/keys.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Isovalent */ + +#include +#include +#include +#include +#include +#include +#include + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "bpf." + +static struct key *bpf_keyring; + +static bool bpf_keyring_unsealed __ro_after_init; +module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); +MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); + +struct bpf_key *bpf_lookup_keyring(void) +{ + struct bpf_key *bkey; + + if (!bpf_keyring) + return NULL; + if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) || + !READ_ONCE(bpf_keyring->restrict_link)) + return NULL; + + bkey = kmalloc_obj(*bkey); + if (!bkey) + return NULL; + + bkey->key = bpf_keyring; + bkey->has_ref = false; + return bkey; +} + +static int __init bpf_keyring_init(void) +{ + struct key *keyring; + + keyring = keyring_alloc(".bpf", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | + KEY_USR_WRITE | KEY_USR_SEARCH | + KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(keyring)) { + pr_err("bpf: cannot allocate bpf keyring: %ld\n", + PTR_ERR(keyring)); + return 0; + } + if (!bpf_keyring_unsealed && + keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) { + pr_err("bpf: cannot seal bpf keyring\n"); + key_revoke(keyring); + key_put(keyring); + return 0; + } + + bpf_keyring = keyring; + key_register_bpf_keyring(keyring); + return 0; +} +late_initcall(bpf_keyring_init); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a725e3917976..0c229a226eab 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -21127,6 +21128,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) return BPF_SIG_KEYRING_SECONDARY; case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: return BPF_SIG_KEYRING_PLATFORM; + case KEY_SPEC_BPF_KEYRING: + return BPF_SIG_KEYRING_BPF; default: return BPF_SIG_KEYRING_USER; } @@ -21164,9 +21167,15 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, return -EINVAL; if (system_keyring_id_check(attr->keyring_id) == 0) key = bpf_lookup_system_key(attr->keyring_id); + else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) + key = bpf_lookup_keyring(); else key = bpf_lookup_user_key(attr->keyring_id, 0); if (!key) { + if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + verbose(env, "the bpf keyring is empty or has not been restricted\n"); + return -ENOKEY; + } verbose(env, "cannot resolve signing keyring with keyring_id %d\n", attr->keyring_id); return -EINVAL; diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index a63c46bb2d14..44358388e395 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -22,6 +22,9 @@ /* Session keyring create vs join semaphore */ static DEFINE_MUTEX(key_session_mutex); +/* BPF keyring reachable through KEY_SPEC_BPF_KEYRING */ +static struct key *bpf_keyring __ro_after_init; + /* The root user's tracking struct */ struct key_user root_key_user = { .usage = REFCOUNT_INIT(3), @@ -590,6 +593,20 @@ bool lookup_user_key_possessed(const struct key *key, return key == match_data->raw_data; } +/** + * key_register_bpf_keyring - Publish the BPF keyring for KEY_SPEC_BPF_KEYRING + * @keyring: The keyring to publish + * + * Make @keyring reachable by userspace through the KEY_SPEC_BPF_KEYRING + * special key ID, so that provisioning it does not require scraping its + * serial out of /proc/keys first. Called once, from an initcall, and never + * undone. + */ +void key_register_bpf_keyring(struct key *keyring) +{ + bpf_keyring = keyring; +} + /* * Look up a key ID given us by userspace with a given permissions mask to get * the key it refers to. @@ -741,6 +758,14 @@ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, key_ref = make_key_ref(key, 1); break; + case KEY_SPEC_BPF_KEYRING: + key = bpf_keyring; + if (!key) + goto error; + __key_get(key); + key_ref = make_key_ref(key, 0); + break; + default: key_ref = ERR_PTR(-EINVAL); if (id < 1) From abaa0835b4f381067cc04bfd58180679d813578b Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:18 +0200 Subject: [PATCH 56/90] bpf: Refuse caller-supplied keyrings when the bpf one is active Nothing changes for systems that do not use the bpf keyring. Without bpf.keyring_unsealed=1 a caller-supplied keyring behaves exactly as before, which also lets it serve as the staging step for software installed onto a running system whose signing key is not enrolled anywhere yet. Passing bpf.keyring_unsealed=1 states that the bpf keyring is the trust anchor for this boot, so from the first program load onwards a caller- supplied keyring is refused with -EPERM. Deriving this from the boot flag rather than from the keyring's runtime state keeps the decision immutable from userspace. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-3-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/admin-guide/kernel-parameters.txt | 8 ++++++++ include/linux/bpf.h | 6 ++++++ kernel/bpf/keys.c | 5 +++++ kernel/bpf/verifier.c | 10 +++++++--- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2beb61092bb3..e70fb15757d4 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -673,6 +673,14 @@ Kernel parameters keys. Once the keyring is restricted it becomes active and can be used for BPF program signature verification. + Setting this also means that the bpf keyring is the + only non-system keyring a loader may select for the + rest of the boot: caller-supplied user/session + keyrings are refused with -EPERM, whether or not + provisioning actually completed. The system keyrings + stay selectable. Leaving it unset keeps the prior + behaviour, where a caller-supplied keyring is allowed. + See Documentation/bpf/signing.rst bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 039674b2f875..3a7eb2185c35 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3821,6 +3821,7 @@ struct bpf_key { struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); struct bpf_key *bpf_lookup_keyring(void); +bool bpf_keyring_enforced(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3846,6 +3847,11 @@ static inline struct bpf_key *bpf_lookup_keyring(void) return NULL; } +static inline bool bpf_keyring_enforced(void) +{ + return false; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c index 08549b3220c1..bffc356839ac 100644 --- a/kernel/bpf/keys.c +++ b/kernel/bpf/keys.c @@ -18,6 +18,11 @@ static bool bpf_keyring_unsealed __ro_after_init; module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); +bool bpf_keyring_enforced(void) +{ + return bpf_keyring_unsealed; +} + struct bpf_key *bpf_lookup_keyring(void) { struct bpf_key *bkey; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c229a226eab..dab618836299 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21165,12 +21165,16 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, if (!attr->signature_size || attr->signature_size > KMALLOC_MAX_CACHE_SIZE) return -EINVAL; - if (system_keyring_id_check(attr->keyring_id) == 0) + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); - else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) + } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { key = bpf_lookup_keyring(); - else + } else if (bpf_keyring_enforced()) { + verbose(env, "caller-supplied keyring refused, use bpf keyring\n"); + return -EPERM; + } else { key = bpf_lookup_user_key(attr->keyring_id, 0); + } if (!key) { if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { verbose(env, "the bpf keyring is empty or has not been restricted\n"); From 300b348e6d5e8e96c065c42c9999baca224c1661 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:19 +0200 Subject: [PATCH 57/90] bpf: Raise the bound on a program's signature size signature_size is bounded by KMALLOC_MAX_CACHE_SIZE, which is 8 KiB on a 4 KiB page system. Back then we chose it somewhat arbitrarily and was picked when a BPF program signature was RSA or ECDSA. ML-DSA (FIPS-204) verification is wired through the X.509 and PKCS#7 parsers, and BPF reaches them too via verify_pkcs7_signature() without having to know the concrete algorithm. The bound becomes a bit too small, thus add an explicit BPF_PROG_MAX_SIGNATURE_SIZE of 64 KiB and use that instead to cover all options. KMALLOC_MAX_CACHE_SIZE is PAGE_SIZE-derived, so what was accepted so far depended on the page size which is not optimal as it should be the same behavior on every configuration. On 64 KiB page kernels this lowers the ceiling from 128 KiB to 64 KiB. Nothing that could have been verified is affected as the largest signature the kernel implements is ML-DSA-87 at 4627 bytes. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-4-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 15 ++++++++++----- .../selftests/bpf/prog_tests/signed_loader.c | 5 +++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index dab618836299..8f585ceb2cd5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21119,6 +21119,14 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +/* + * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably + * above the largest signature the kernel can verify, and far below anything + * that would make rejecting a load expensive. Deliberately a fixed number so + * that what the syscall accepts does not depend on PAGE_SIZE. + */ +#define BPF_PROG_MAX_SIGNATURE_SIZE (64 * 1024) + static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) { switch (keyring_id) { @@ -21158,13 +21166,10 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, u64 data_sz; int err = 0; - /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. - */ if (!attr->signature_size || - attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE) return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 77381d345435..0c5294738d6c 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -571,8 +571,9 @@ static void signature_too_large(void) if (gen_loader_fixture_init(&f) == 0) { /* - * signature_size beyond the kernel's bound (KMALLOC_MAX_CACHE_SIZE) - * is rejected before the buffer is read. + * signature_size beyond the kernel's bound + * (BPF_PROG_MAX_SIGNATURE_SIZE) is rejected before the buffer + * is read. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); From d37168c99361b6ee36d5be0c1e159b594f32473c Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:20 +0200 Subject: [PATCH 58/90] bpftool: Support ML-DSA program signing Add bpftool support for ML-DSA program signing and drop the flag for ML-DSA keys on affected OpenSSL versions, the same way as commit 0ad9a71933e7 ("modsign: Enable ML-DSA module signing"). Also, an ML-DSA-87 signature is 4627 bytes on its own, so the blob does not fit into the 4 KiB of MAX_SIG_SIZE anymore and signing would fail otherwise. Bump to 16 KiB. MAX_SIG_SIZE only sizes the buffer for what bpftool itself emits (unrelated to BPF_PROG_MAX_SIGNATURE_SIZE). Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-5-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/main.h | 2 +- tools/bpf/bpftool/sign.c | 24 +++++++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 78b6e0ebb85d..9315a1db1f7c 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -57,7 +57,7 @@ static inline void *u64_to_ptr(__u64 ptr) }) #define ERR_MAX_LEN 1024 -#define MAX_SIG_SIZE 4096 +#define MAX_SIG_SIZE 16384 #define BPF_TAG_FMT "%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx" diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index 88726a6db6d0..adbfd4ae5c02 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c @@ -130,6 +130,12 @@ __u32 register_session_key(const char *key_der_path) int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) { + unsigned int signer_flags = CMS_NOCERTS | CMS_BINARY | CMS_NOSMIMECAP | +#ifdef CMS_NO_SIGNING_TIME + CMS_NO_SIGNING_TIME | +#endif + CMS_USE_KEYID | CMS_NOATTR; + const EVP_MD *cms_digest = EVP_sha256(); BIO *bd_in = NULL, *bd_out = NULL; EVP_PKEY *private_key = NULL; CMS_ContentInfo *cms = NULL; @@ -167,6 +173,20 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } +#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L + if (EVP_PKEY_is_a(private_key, "ML-DSA-44") || + EVP_PKEY_is_a(private_key, "ML-DSA-65") || + EVP_PKEY_is_a(private_key, "ML-DSA-87")) { + /* + * See workaround in 0ad9a71933e7 ("modsign: Enable ML-DSA + * module signing"). Kernel only accepts sha512, see also + * 8bbdeb7a25b4 ("pkcs7, x509: Add ML-DSA support"). + */ + signer_flags &= ~CMS_NOATTR; + cms_digest = EVP_sha512(); + } +#endif + cms = CMS_sign(NULL, NULL, NULL, NULL, CMS_NOCERTS | CMS_PARTIAL | CMS_BINARY | CMS_DETACHED | CMS_STREAM); @@ -175,9 +195,7 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } - if (!CMS_add1_signer(cms, x509, private_key, EVP_sha256(), - CMS_NOCERTS | CMS_BINARY | CMS_NOSMIMECAP | - CMS_USE_KEYID | CMS_NOATTR)) { + if (!CMS_add1_signer(cms, x509, private_key, cms_digest, signer_flags)) { err = -EINVAL; goto cleanup; } From 6754aeca602224a3f5e8c6703c6549791d8f6227 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:21 +0200 Subject: [PATCH 59/90] selftests/bpf: Add a test for the sealed bpf keyring bpf_keyring_sealed checks that a load naming the bpf keyring fails while the keyring has not been provisioned. It uses a junk signature as the size check and the keyring lookup both happen before any crypto, so the error under test is reached without a real signature and the ordering is what gets verified: # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t signed_loader [...] #425/9 signed_loader/signed_module_kfunc_rejected:OK #425/10 signed_loader/signature_failure_logs:OK #425/11 signed_loader/signature_too_large:OK #425/12 signed_loader/signature_zero_size:OK #425/13 signed_loader/signature_bad_keyring:OK #425/14 signed_loader/bpf_keyring_sealed:OK #425/15 signed_loader/metadata_ctx_max_entries_ignored:OK #425/16 signed_loader/metadata_ctx_initial_value_ignored:OK #425/17 signed_loader/signature_authenticates_insns:OK #425/18 signed_loader/signature_authenticates_metadata:OK #425/19 signed_loader/hash_requires_frozen:OK [...] #425 signed_loader:OK Summary: 1/30 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-6-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/signed_loader.c | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 0c5294738d6c..e30166c00be6 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -32,8 +32,13 @@ enum { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; +#ifndef KEY_SPEC_BPF_KEYRING +#define KEY_SPEC_BPF_KEYRING -9 +#endif + static int load_loader(const void *insns, __u32 insns_sz, int map_fd, const void *sig, __u32 sig_sz, __s32 keyring_id, __u32 fd_array_cnt) @@ -627,6 +632,83 @@ static void signature_bad_keyring(void) gen_loader_fixture_fini(&f); } +static bool keyring_unsealed_boot(void) +{ + char val = 0; + int fd; + + fd = open("/sys/module/bpf/parameters/keyring_unsealed", O_RDONLY); + if (fd < 0) + return false; + if (read(fd, &val, 1) != 1) + val = 0; + close(fd); + return val == 'Y' || val == '1'; +} + +static int bpf_keyring_lookup(int *nr_keys) +{ + char line[512], type[32], desc[64]; + int serial = -ENOENT; + FILE *f; + + f = fopen("/proc/keys", "r"); + if (!f) + return -errno; + + while (fgets(line, sizeof(line), f)) { + unsigned int hex; + char *sum; + + if (sscanf(line, "%x %*s %*s %*s %*s %*s %*s %31s %63s", + &hex, type, desc) != 3) + continue; + if (strcmp(type, "keyring") || strcmp(desc, ".bpf:")) + continue; + + serial = (int)hex; + if (nr_keys) { + sum = strstr(line, ".bpf: "); + *nr_keys = !sum || !strncmp(sum + 6, "empty", 5) ? + 0 : atoi(sum + 6); + } + break; + } + fclose(f); + return serial; +} + +static void bpf_keyring_sealed(void) +{ + static const __u8 junk[64] = {}; + struct gen_loader_fixture f; + int serial, key, fd; + + if (keyring_unsealed_boot()) { + printf("%s:SKIP:the bpf keyring was unsealed at boot\n", __func__); + test__skip(); + return; + } + serial = bpf_keyring_lookup(NULL); + if (serial >= 0) { + key = syscall(__NR_add_key, "user", "sealprobe", "x", 1, + KEY_SPEC_BPF_KEYRING); + if (key >= 0) + syscall(__NR_keyctl, KEYCTL_UNLINK, key, + KEY_SPEC_BPF_KEYRING); + ASSERT_EQ(key < 0 ? -errno : 0, -EPERM, + "nothing links into a sealed keyring"); + } + if (gen_loader_fixture_init(&f) == 0) { + fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_BPF_KEYRING, 0); + ASSERT_EQ(fd, -ENOKEY, "sealed bpf keyring rejected"); + if (fd >= 0) + close(fd); + } + gen_loader_fixture_fini(&f); +} + /* * A signed loader must ignore ctx-supplied map dimensions: the host cannot * resize a signed program's maps via the loader ctx. Drive a one-map program @@ -1806,6 +1888,8 @@ void test_signed_loader(void) signature_zero_size(); if (test__start_subtest("signature_bad_keyring")) signature_bad_keyring(); + if (test__start_subtest("bpf_keyring_sealed")) + bpf_keyring_sealed(); if (test__start_subtest("metadata_ctx_max_entries_ignored")) metadata_ctx_max_entries_ignored(); if (test__start_subtest("metadata_ctx_initial_value_ignored")) From e7ada0800ce2a6dcc35ca0e1dfdab37bfb2ea4a9 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:22 +0200 Subject: [PATCH 60/90] selftests/bpf: Rebuild signed lskels when signing key changes The signing key is regenerated whenever verify_sig_setup.sh changes, but the signed light skeletons only depend on the BPF object and on bpftool, not on the key they are signed with. Thus, add the certificate as a prereq so a new key forces the skeletons to be signed again. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-7-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 5f1a3bfc0569..05b3ee64290f 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -663,7 +663,7 @@ $(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) }) && \ rm -f $$(<:.o=.llinked1.o) $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o) -$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT) +$(TRUNNER_BPF_LSKELS_SIGNED): %.lskel.h: %.bpf.o $(BPFTOOL) $(VERIFICATION_CERT) | $(TRUNNER_OUTPUT) $(Q)$(if $(PERMISSIVE),if [ ! -f $$< ]; then \ $$(RM) $$@; \ printf ' %-12s %s\n' 'SKIP-SKEL' '$$(notdir $$@)' 1>&2; \ From 1841d6cbc5c777d5dfba3723965ab14a9764d96e Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:23 +0200 Subject: [PATCH 61/90] selftests/bpf: Rename the verify_sig_setup.sh setup into setup-rsa The script's "setup" action generates an RSA key, enrolls it and builds a keyring around it. The name says nothing about the algorithm, which is fine while there is only one, but we'll add "setup-mldsa" soon, therefore rename the existing one into "setup-rsa". No functional change. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-8-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/signed_loader.c | 18 +++++++++--------- .../bpf/prog_tests/verify_pkcs7_sig.c | 4 ++-- .../testing/selftests/bpf/verify_sig_setup.sh | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index e30166c00be6..fbf14c7dd560 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -462,7 +462,7 @@ static void signed_btf_fd_array_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -914,7 +914,7 @@ static void signature_authenticates_insns(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1014,7 +1014,7 @@ static void signature_authenticates_metadata(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1350,7 +1350,7 @@ static void lsm_signature_verdict(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); dir = NULL; goto out; @@ -1533,7 +1533,7 @@ static void loadtime_verify(struct bpf_object *obj, int expect_maps) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1631,7 +1631,7 @@ static void signed_no_fd_array(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } @@ -1702,7 +1702,7 @@ static void signed_map_by_fd_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out_map; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); goto out_map; } @@ -1764,7 +1764,7 @@ static void signed_sparse_fd_array_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) goto out_map; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); goto out_map; } @@ -1818,7 +1818,7 @@ static void signed_module_kfunc_rejected(void) dir = mkdtemp(dir_tmpl); if (!ASSERT_OK_PTR(dir, "mkdtemp")) return; - if (!ASSERT_OK(run_setup("setup", dir), "verify_sig_setup")) { + if (!ASSERT_OK(run_setup("setup-rsa", dir), "verify_sig_setup")) { rmdir(dir); return; } diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c index f327feb8e38c..12b146d205d7 100644 --- a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c +++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c @@ -257,7 +257,7 @@ static void test_verify_pkcs7_sig_from_map(void) if (!ASSERT_OK_PTR(tmp_dir, "mkdtemp")) return; - ret = _run_setup_process(tmp_dir, "setup"); + ret = _run_setup_process(tmp_dir, "setup-rsa"); if (!ASSERT_OK(ret, "_run_setup_process")) goto close_prog; @@ -458,7 +458,7 @@ static void test_pkcs7_sig_fsverity(void) snprintf(data_path, PATH_MAX, "%s/data-file", tmp_dir); snprintf(sig_path, PATH_MAX, "%s/sig-file", tmp_dir); - ret = _run_setup_process(tmp_dir, "setup"); + ret = _run_setup_process(tmp_dir, "setup-rsa"); if (!ASSERT_OK(ret, "_run_setup_process")) goto out; diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh index 09179fb551f0..202e6e6418fe 100755 --- a/tools/testing/selftests/bpf/verify_sig_setup.sh +++ b/tools/testing/selftests/bpf/verify_sig_setup.sh @@ -28,7 +28,7 @@ authorityKeyIdentifier=keyid usage() { - echo "Usage: $0 " + echo "Usage: $0 " exit 1 } @@ -47,7 +47,7 @@ genkey() ${tmp_dir}/signing_key.der -outform der } -setup() +setup_rsa() { local tmp_dir="$1" @@ -108,8 +108,8 @@ main() [[ ! -d "${tmp_dir}" ]] && echo "Directory ${tmp_dir} doesn't exist" && exit 1 - if [[ "${action}" == "setup" ]]; then - setup "${tmp_dir}" + if [[ "${action}" == "setup-rsa" ]]; then + setup_rsa "${tmp_dir}" elif [[ "${action}" == "genkey" ]]; then genkey "${tmp_dir}" elif [[ "${action}" == "cleanup" ]]; then From 478cf384def8cc4cb7f92739b73c76acbac976ca Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:24 +0200 Subject: [PATCH 62/90] selftests/bpf: Add an end-to-end ML-DSA signed loader test The BPF signing is algorithm agnostic, but so far the BPF CI only has tested a single one. BPF hands verify_pkcs7_signature() a keyring and byte ranges, and everything below it already understands ML-DSA, so add a test for ML-DSA signed program to validate it works as well. # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t signed_loader [...] #425/10 signed_loader/signature_failure_logs:OK #425/11 signed_loader/signature_too_large:OK #425/12 signed_loader/signature_zero_size:OK #425/13 signed_loader/signature_bad_keyring:OK #425/14 signed_loader/bpf_keyring_sealed:OK #425/15 signed_loader/mldsa_signed_load:OK #425/16 signed_loader/metadata_ctx_max_entries_ignored:OK #425/17 signed_loader/metadata_ctx_initial_value_ignored:OK #425/18 signed_loader/signature_authenticates_insns:OK #425/19 signed_loader/signature_authenticates_metadata:OK #425/20 signed_loader/hash_requires_frozen:OK [...] #425 signed_loader:OK Summary: 1/31 PASSED, 0 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-9-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/config | 2 + .../selftests/bpf/prog_tests/signed_loader.c | 125 +++++++++++++++++- .../testing/selftests/bpf/verify_sig_setup.sh | 57 +++++++- 3 files changed, 177 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config index ea7044f30adc..2f79688dcf7c 100644 --- a/tools/testing/selftests/bpf/config +++ b/tools/testing/selftests/bpf/config @@ -12,7 +12,9 @@ CONFIG_BPF_SYSCALL=y # CONFIG_BPF_UNPRIV_DEFAULT_OFF is not set CONFIG_CGROUP_BPF=y CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_MLDSA=y CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y CONFIG_CRYPTO_USER_API=y CONFIG_CRYPTO_USER_API_HASH=y CONFIG_CRYPTO_USER_API_SKCIPHER=y diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index fbf14c7dd560..87f155de0e65 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -39,6 +39,12 @@ enum { #define KEY_SPEC_BPF_KEYRING -9 #endif +/* verify_sig_setup.sh exits with this when openssl cannot do ML-DSA. */ +#define SETUP_SKIP (-77) + +/* FIPS-204 ML-DSA-87 signature size, see include/crypto/mldsa.h. */ +#define MLDSA87_SIGNATURE_SIZE 4627 + static int load_loader(const void *insns, __u32 insns_sz, int map_fd, const void *sig, __u32 sig_sz, __s32 keyring_id, __u32 fd_array_cnt) @@ -161,12 +167,30 @@ static int run_setup(const char *cmd, const char *dir) } if (waitpid(pid, &status, 0) < 0) return -errno; - return (WIFEXITED(status) && - WEXITSTATUS(status) == 0) ? 0 : -EINVAL; + if (!WIFEXITED(status)) + return -EINVAL; + return -WEXITSTATUS(status); } -static int sign_buf(const char *dir, const void *buf, __u32 len, - void *sig, __u32 *sig_sz) +static void genkey_dir_fini(const char *dir) +{ + static const char * const files[] = { + "signing_key.der", "signing_key.pem", "x509.genkey", + }; + char path[PATH_MAX]; + size_t i; + + if (!dir) + return; + for (i = 0; i < ARRAY_SIZE(files); i++) { + snprintf(path, sizeof(path), "%s/%s", dir, files[i]); + unlink(path); + } + rmdir(dir); +} + +static int sign_buf_digest(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz, const char *digest) { char data_tmpl[PATH_MAX], key[PATH_MAX]; char sigpath[PATH_MAX + sizeof(".p7s")]; @@ -195,7 +219,7 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, } if (pid == 0) { snprintf(key, sizeof(key), "%s/signing_key.pem", dir); - execlp("./sign-file", "./sign-file", "-d", "sha256", + execlp("./sign-file", "./sign-file", "-d", digest, key, key, data_tmpl, NULL); exit(1); } @@ -233,6 +257,12 @@ static int sign_buf(const char *dir, const void *buf, __u32 len, return ret; } +static int sign_buf(const char *dir, const void *buf, __u32 len, + void *sig, __u32 *sig_sz) +{ + return sign_buf_digest(dir, buf, len, sig, sig_sz, "sha256"); +} + struct gen_loader_fixture { struct test_signed_loader *skel; struct gen_loader_opts gopts; @@ -1607,6 +1637,89 @@ static void loadtime_with_map(void) test_signed_loader_map__destroy(skel); } +/* + * End-to-end signed load with a post-quantum key. ML-DSA (FIPS-204) is wired + * through the X.509 and PKCS#7 parsers, and BPF reaches them via + * verify_pkcs7_signature() without knowing the algorithm, so an ML-DSA key in + * the keyring should verify an ML-DSA signed program with no BPF-side work. + */ +static void mldsa_signed_load(void) +{ + char dir_tmpl[] = "/tmp/bpfmldsaXXXXXX"; + int map_fd = -1, prog_fd = -1, err; + __u8 *sig = NULL, *buf = NULL; + struct gen_loader_fixture f; + bool have_fixture = false; + __u32 sig_sz = 16384; + char *dir; + + syscall(__NR_request_key, "keyring", "_uid.0", NULL, + KEY_SPEC_SESSION_KEYRING); + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + + err = run_setup("setup-mldsa", dir); + if (err == SETUP_SKIP) { + printf("%s:SKIP:no working ML-DSA signing, set SELFTESTS_VERBOSE=1\n", + __func__); + test__skip(); + genkey_dir_fini(dir); + return; + } + if (!ASSERT_OK(err, "verify_sig_setup setup-mldsa")) { + genkey_dir_fini(dir); + return; + } + + sig = malloc(sig_sz); + if (!ASSERT_OK_PTR(sig, "sig buf")) + goto out; + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + + /* + * ML-DSA hashes the message itself, but openssl before 4.0 cannot + * produce a CMS message without signedAttrs for it, and with those in + * play only SHA-512 is permitted for the messageDigest attribute. + */ + if (!ASSERT_OK(sign_buf_digest(dir, buf, f.gopts.insns_sz + f.data_sz, + sig, &sig_sz, "sha512"), + "sign insns||metadata with ML-DSA")) + goto out; + + /* + * Guard against the setup silently handing back some other key type: + * an RSA or ECDSA signature is a few hundred bytes, where an ML-DSA-87 + * one cannot be smaller than the raw signature it carries. + */ + ASSERT_GT(sig_sz, MLDSA87_SIGNATURE_SIZE, "ML-DSA-87 signature size"); + + map_fd = setup_meta_map(&f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + goto out; + prog_fd = load_loader(f.gopts.insns, f.gopts.insns_sz, map_fd, sig, + sig_sz, KEY_SPEC_SESSION_KEYRING, 1); + ASSERT_OK_FD(prog_fd, "ML-DSA signed loader load"); +out: + if (prog_fd >= 0) + close(prog_fd); + if (map_fd >= 0) + close(map_fd); + if (have_fixture) + gen_loader_fixture_fini(&f); + free(buf); + free(sig); + run_setup("cleanup", dir); +} + /* * A signed program need not bind any map. A plain BPF_PROG_TYPE_SYSCALL * program with no fd_array is signed over its instructions alone: the kernel @@ -1890,6 +2003,8 @@ void test_signed_loader(void) signature_bad_keyring(); if (test__start_subtest("bpf_keyring_sealed")) bpf_keyring_sealed(); + if (test__start_subtest("mldsa_signed_load")) + mldsa_signed_load(); if (test__start_subtest("metadata_ctx_max_entries_ignored")) metadata_ctx_max_entries_ignored(); if (test__start_subtest("metadata_ctx_initial_value_ignored")) diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh index 202e6e6418fe..ba5921e4b12d 100755 --- a/tools/testing/selftests/bpf/verify_sig_setup.sh +++ b/tools/testing/selftests/bpf/verify_sig_setup.sh @@ -28,7 +28,7 @@ authorityKeyIdentifier=keyid usage() { - echo "Usage: $0 " + echo "Usage: $0 " exit 1 } @@ -57,6 +57,57 @@ setup_rsa() keyctl link $key_id $keyring_id } +mldsa_supported() +{ + local tmp_dir="$1" + + genkey_mldsa "${tmp_dir}" || return 1 + : > ${tmp_dir}/probe + # Same digest as the caller signs with, see sign_buf_digest(). + ./sign-file -d sha512 ${tmp_dir}/signing_key.pem \ + ${tmp_dir}/signing_key.pem ${tmp_dir}/probe || return 1 + rm -f ${tmp_dir}/probe ${tmp_dir}/probe.p7s +} + +genkey_mldsa() +{ + local tmp_dir="$1" + + echo "${x509_genkey_content}" > ${tmp_dir}/x509.genkey + + # No - here: ML-DSA hashes the message itself, so openssl + # ignores an explicit digest for it. + openssl req -new -nodes -utf8 -days 36500 \ + -batch -x509 -newkey ML-DSA-87 \ + -config ${tmp_dir}/x509.genkey \ + -outform PEM -out ${tmp_dir}/signing_key.pem \ + -keyout ${tmp_dir}/signing_key.pem 2>&1 + + openssl x509 -in ${tmp_dir}/signing_key.pem -out \ + ${tmp_dir}/signing_key.der -outform der +} + +mldsa_skip() +{ + local tmp_dir="$1" + + rm -f ${tmp_dir}/x509.genkey ${tmp_dir}/signing_key.pem \ + ${tmp_dir}/signing_key.der ${tmp_dir}/probe \ + ${tmp_dir}/probe.p7s + exit 77 +} + +setup_mldsa() +{ + local tmp_dir="$1" + + mldsa_supported "${tmp_dir}" || mldsa_skip "${tmp_dir}" + key_id=$(cat ${tmp_dir}/signing_key.der | + keyctl padd asymmetric ebpf_testing_key @s) + keyring_id=$(keyctl newring ebpf_testing_keyring @s) + keyctl link $key_id $keyring_id +} + cleanup() { local tmp_dir="$1" @@ -91,7 +142,7 @@ catch() local exit_code="$1" local log_file="$2" - if [[ "${exit_code}" -ne 0 ]]; then + if [[ "${exit_code}" -ne 0 && "${exit_code}" -ne 77 ]]; then cat "${log_file}" >&3 fi @@ -110,6 +161,8 @@ main() if [[ "${action}" == "setup-rsa" ]]; then setup_rsa "${tmp_dir}" + elif [[ "${action}" == "setup-mldsa" ]]; then + setup_mldsa "${tmp_dir}" elif [[ "${action}" == "genkey" ]]; then genkey "${tmp_dir}" elif [[ "${action}" == "cleanup" ]]; then From b638a82d64f1ee480c4b6c6c8e3da2ae84c1edb0 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:25 +0200 Subject: [PATCH 63/90] selftests/bpf: Allow appending to guest kernel cmdline in vmtest.sh vmtest.sh currently hardcodes the guest command line, so there is no way to ask for such a setting without editing the script. Append $KERNEL_CMDLINE_EXTRA when set so it can be used for testing the BPF keyring: # KERNEL_CMDLINE_EXTRA="bpf.keyring_unsealed=1" \ ./vmtest.sh -- ./test_progs -t signed_loader Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-10-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/vmtest.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 6a3d026d76bd..e7e0b419a0b8 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -107,6 +107,14 @@ Options: -s) Instead of powering off the VM, start an interactive shell. If is specified, the shell runs after the command finishes executing + +Environment variables: + + KERNEL_CMDLINE_EXTRA + Extra arguments to append to the guest kernel command + line, for tests that need a boot-time setting. e.g: + + KERNEL_CMDLINE_EXTRA="bpf.keyring_unsealed=1" $0 -- ./test_progs -t signed_loader EOF } @@ -286,6 +294,12 @@ EOF QEMU_FLAGS=("${HOST_FLAGS[@]}") fi + local kernel_cmdline="root=/dev/vda rw console=${QEMU_CONSOLE}" + + if [[ -n "${KERNEL_CMDLINE_EXTRA:-}" ]]; then + kernel_cmdline+=" ${KERNEL_CMDLINE_EXTRA}" + fi + ${QEMU_BINARY} \ -nodefaults \ -display none \ @@ -294,7 +308,7 @@ EOF -m 4G \ -drive file="${rootfs_img}",format=raw,index=1,media=disk,if=virtio,cache=none \ -kernel "${kernel_bzimage}" \ - -append "root=/dev/vda rw console=${QEMU_CONSOLE}" + -append "${kernel_cmdline}" } copy_logs() From 1c8cb8bf73293264e2a44d0b029a5047a7dd5e20 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:26 +0200 Subject: [PATCH 64/90] selftests/bpf: Add tests for bpf keyring in signed loader bpf_keyring_provisioned walks the keyring through its whole lifecycle in one boot for ease of testing. It enrolls a freshly generated key into the bpf keyring, confirms a load is still refused with -ENOKEY while the keyring carries no restriction, then restricts it, and only then does the same signed BPF program load with the bpf keyring. A caller-supplied keyring is asserted to be refused both before and after the restriction, since what refuses it is bpf.keyring_unsealed=1 rather than the state of the keyring. Unsealing is a boot-time decision which also refuses the caller-supplied keyrings that most subtests here sign against, so each subtest is tagged with the boot it needs and the ones which cannot run report as skipped instead of being dropped. Regular run: # LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t signed_loader [...] #425/12 signed_loader/signature_zero_size:OK #425/13 signed_loader/signature_bad_keyring:OK #425/14 signed_loader/bpf_keyring_sealed:OK [...] #425/30 signed_loader/signed_map_by_fd_rejected:OK #425/31 signed_loader/signed_sparse_fd_array_rejected:OK #425/32 signed_loader/bpf_keyring_provisioned:SKIP #425 signed_loader:OK (SKIP: 1/32) Summary: 1/31 PASSED, 1 SKIPPED, 0/0 FAILED Unsealed run: # KERNEL_CMDLINE_EXTRA="bpf.keyring_unsealed=1" \ LDLIBS=-static PKG_CONFIG='pkg-config --static' ./vmtest.sh -- ./test_progs -t signed_loader #425/1 signed_loader/loadtime_no_map:SKIP #425/2 signed_loader/loadtime_with_map:SKIP #425/3 signed_loader/metadata_match:OK [...] #425/30 signed_loader/signed_map_by_fd_rejected:SKIP #425/31 signed_loader/signed_sparse_fd_array_rejected:SKIP #425/32 signed_loader/bpf_keyring_provisioned:OK #425 signed_loader:OK (SKIP: 17/32) Summary: 1/15 PASSED, 17 SKIPPED, 0/0 FAILED Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-11-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/signed_loader.c | 376 +++++++++++++----- 1 file changed, 287 insertions(+), 89 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 87f155de0e65..a0f93756e717 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -45,9 +45,9 @@ enum { /* FIPS-204 ML-DSA-87 signature size, see include/crypto/mldsa.h. */ #define MLDSA87_SIGNATURE_SIZE 4627 -static int load_loader(const void *insns, __u32 insns_sz, int map_fd, - const void *sig, __u32 sig_sz, __s32 keyring_id, - __u32 fd_array_cnt) +static int load_loader_log(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt, char *log_buf, __u32 log_sz) { union bpf_attr attr; int fd; @@ -59,18 +59,31 @@ static int load_loader(const void *insns, __u32 insns_sz, int map_fd, attr.license = ptr_to_u64("Dual BSD/GPL"); attr.prog_flags = BPF_F_SLEEPABLE; attr.fd_array = ptr_to_u64(&map_fd); + attr.fd_array_cnt = fd_array_cnt; if (sig) { attr.signature = ptr_to_u64(sig); attr.signature_size = sig_sz; attr.keyring_id = keyring_id; } - attr.fd_array_cnt = fd_array_cnt; + if (log_buf) { + attr.log_level = 1; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = log_sz; + } memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, offsetofend(union bpf_attr, keyring_id)); return fd < 0 ? -errno : fd; } +static int load_loader(const void *insns, __u32 insns_sz, int map_fd, + const void *sig, __u32 sig_sz, __s32 keyring_id, + __u32 fd_array_cnt) +{ + return load_loader_log(insns, insns_sz, map_fd, sig, sig_sz, keyring_id, + fd_array_cnt, NULL, 0); +} + static int run_gen_loader(const void *insns, __u32 insns_sz, const void *data, __u32 data_sz, const void *excl, __u32 excl_sz, @@ -205,6 +218,7 @@ static int sign_buf_digest(const char *dir, const void *buf, __u32 len, fd = mkstemp(data_tmpl); if (fd < 0) return -errno; + snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); if (write(fd, buf, len) != (ssize_t)len) { close(fd); ret = -EIO; @@ -229,30 +243,28 @@ static int sign_buf_digest(const char *dir, const void *buf, __u32 len, goto out; } - snprintf(sigpath, sizeof(sigpath), "%s.p7s", data_tmpl); if (stat(sigpath, &st) < 0) { ret = -errno; goto out; } if (st.st_size > (off_t)*sig_sz) { ret = -E2BIG; - goto out_sig; + goto out; } fd = open(sigpath, O_RDONLY); if (fd < 0) { ret = -errno; - goto out_sig; + goto out; } if (read(fd, sig, st.st_size) != st.st_size) { close(fd); ret = -EIO; - goto out_sig; + goto out; } close(fd); *sig_sz = st.st_size; -out_sig: - unlink(sigpath); out: + unlink(sigpath); unlink(data_tmpl); return ret; } @@ -564,7 +576,6 @@ static void signature_failure_logs(void) static const __u8 junk[64] = { 0x30, 0x42, 0x13, 0x37, }; char log_buf[1024] = {}; struct gen_loader_fixture f; - union bpf_attr attr; int fd; if (gen_loader_fixture_init(&f) == 0) { @@ -573,22 +584,9 @@ static void signature_failure_logs(void) * failure is reported through the verifier log. A present-but- * invalid signature is rejected and the log says why. */ - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_SYSCALL; - attr.insns = ptr_to_u64(f.gopts.insns); - attr.insn_cnt = f.gopts.insns_sz / sizeof(struct bpf_insn); - attr.license = ptr_to_u64("Dual BSD/GPL"); - attr.prog_flags = BPF_F_SLEEPABLE; - attr.signature = ptr_to_u64(junk); - attr.signature_size = sizeof(junk); - attr.keyring_id = KEY_SPEC_SESSION_KEYRING; - attr.log_level = 1; - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = sizeof(log_buf); - memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); - - fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, - offsetofend(union bpf_attr, keyring_id)); + fd = load_loader_log(f.gopts.insns, f.gopts.insns_sz, -1, junk, + sizeof(junk), KEY_SPEC_SESSION_KEYRING, 0, + log_buf, sizeof(log_buf)); ASSERT_LT(fd, 0, "invalid signature rejected at load"); if (fd >= 0) close(fd); @@ -708,6 +706,23 @@ static int bpf_keyring_lookup(int *nr_keys) return serial; } +static long keyctl_ret(int cmd, unsigned long arg2, unsigned long arg3) +{ + long ret = syscall(__NR_keyctl, cmd, arg2, arg3); + + return ret < 0 ? -errno : ret; +} + +/* + * What the bpf keyring still needs once it got provisioned: KEY_POS_SEARCH + * for the in-kernel search during verification, and the user view/read bits + * so it stays visible in /proc/keys, rest is dropped so the enrolled is + * therefore final. + */ +#define BPF_KEYRING_PERM_LOCKED 0x08030000 +/* What bpf_keyring_init() grants at boot. */ +#define BPF_KEYRING_PERM_INITIAL 0x082f0000 + static void bpf_keyring_sealed(void) { static const __u8 junk[64] = {}; @@ -721,6 +736,9 @@ static void bpf_keyring_sealed(void) } serial = bpf_keyring_lookup(NULL); if (serial >= 0) { + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, + KEY_SPEC_BPF_KEYRING, 0), serial, + "KEY_SPEC_BPF_KEYRING resolves to the bpf keyring"); key = syscall(__NR_add_key, "user", "sealprobe", "x", 1, KEY_SPEC_BPF_KEYRING); if (key >= 0) @@ -739,6 +757,183 @@ static void bpf_keyring_sealed(void) gen_loader_fixture_fini(&f); } +static int try_load(const struct gen_loader_fixture *f, const void *sig, + __u32 sig_sz, __s32 keyring_id, char *log_buf, __u32 log_sz) +{ + int map_fd, prog_fd; + + map_fd = setup_meta_map(f); + if (!ASSERT_OK_FD(map_fd, "meta_map")) + return map_fd; + prog_fd = load_loader_log(f->gopts.insns, f->gopts.insns_sz, map_fd, + sig, sig_sz, keyring_id, 1, log_buf, log_sz); + close(map_fd); + if (prog_fd >= 0) + close(prog_fd); + return prog_fd; +} + +/* + * This needs bpf.keyring_unsealed=1 on the guest kernel command line, which + * vmtest.sh can pass via KERNEL_CMDLINE_EXTRA. There is no way to unseal the + * keyring from here, so without it the test skips. It also only works once + * per boot, as restricting a keyring cannot be undone. + */ +static void bpf_keyring_provisioned(void) +{ + char dir_tmpl[] = "/tmp/bpfkeyringXXXXXX"; + char bad_tmpl[] = "/tmp/bpfkeyringbadXXXXXX"; + __u8 *sig = NULL, *bad = NULL, *buf = NULL; + int serial, err; + int nr_keys = 0, der_fd = -1; + struct gen_loader_fixture f; + __u32 sig_sz = 8192, bad_sz; + bool have_fixture = false; + char *dir, *bad_dir = NULL; + char log_buf[1024] = {}; + char path[PATH_MAX]; + __u8 der[4096]; + ssize_t der_sz; + + serial = bpf_keyring_lookup(&nr_keys); + if (serial < 0) { + printf("%s:SKIP:no bpf keyring (needs CONFIG_KEYS)\n", __func__); + test__skip(); + return; + } + if (nr_keys != 0) { + printf("%s:SKIP:the bpf keyring has already been provisioned\n", + __func__); + test__skip(); + return; + } + + dir = mkdtemp(dir_tmpl); + if (!ASSERT_OK_PTR(dir, "mkdtemp")) + return; + if (!ASSERT_OK(run_setup("genkey", dir), "verify_sig_setup genkey")) + goto rmdir; + + snprintf(path, sizeof(path), "%s/signing_key.der", dir); + der_fd = open(path, O_RDONLY); + if (!ASSERT_OK_FD(der_fd, "open signing_key.der")) + goto rmdir; + der_sz = read(der_fd, der, sizeof(der)); + close(der_fd); + if (!ASSERT_GT(der_sz, 0, "read signing_key.der")) + goto rmdir; + + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, KEY_SPEC_BPF_KEYRING, 0), + serial, "KEY_SPEC_BPF_KEYRING resolves to the bpf keyring"); + + err = syscall(__NR_add_key, "asymmetric", "", der, (size_t)der_sz, + KEY_SPEC_BPF_KEYRING); + if (err < 0 && errno == EPERM) { + printf("%s:SKIP:the bpf keyring is sealed, need bpf.keyring_unsealed=1\n", + __func__); + test__skip(); + goto rmdir; + } + if (!ASSERT_GE(err, 0, "add the signing key to the bpf keyring")) + goto rmdir; + + sig = malloc(sig_sz); + if (!ASSERT_OK_PTR(sig, "sig buf")) + goto out; + have_fixture = true; + if (gen_loader_fixture_init(&f) != 0) + goto out; + + buf = malloc((size_t)f.gopts.insns_sz + f.data_sz); + if (!ASSERT_OK_PTR(buf, "signbuf")) + goto out; + memcpy(buf, f.gopts.insns, f.gopts.insns_sz); + memcpy(buf + f.gopts.insns_sz, f.blob, f.data_sz); + if (!ASSERT_OK(sign_buf(dir, buf, f.gopts.insns_sz + f.data_sz, sig, + &sig_sz), "sign insns||metadata")) + goto out; + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0), + -ENOKEY, "unrestricted keyring still not consulted"); + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_SESSION_KEYRING, NULL, 0), + -EPERM, "caller-supplied keyring refused before provisioning"); + + if (!ASSERT_OK(syscall(__NR_keyctl, KEYCTL_RESTRICT_KEYRING, + KEY_SPEC_BPF_KEYRING, NULL, NULL), + "restrict bpf keyring")) + goto out; + + if (!ASSERT_OK_FD(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, + NULL, 0), + "load signed by a key in the .bpf keyring")) + goto out; + + bad_dir = mkdtemp(bad_tmpl); + if (!ASSERT_OK_PTR(bad_dir, "mkdtemp unenrolled")) + goto out; + if (!ASSERT_OK(run_setup("genkey", bad_dir), "verify_sig_setup genkey unenrolled")) + goto out; + bad_sz = 8192; + bad = malloc(bad_sz); + if (!ASSERT_OK_PTR(bad, "bad sig buf")) + goto out; + if (!ASSERT_OK(sign_buf(bad_dir, buf, f.gopts.insns_sz + f.data_sz, bad, + &bad_sz), "sign with an unenrolled key")) + goto out; + + ASSERT_EQ(try_load(&f, bad, bad_sz, KEY_SPEC_BPF_KEYRING, log_buf, + sizeof(log_buf)), -ENOKEY, + "key outside the bpf keyring refused"); + ASSERT_HAS_SUBSTR(log_buf, "signature verification failed", + "the bpf keyring was consulted"); + + f.blob[0] ^= 0xff; + err = try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0); + f.blob[0] ^= 0xff; + ASSERT_EQ(err, -EKEYREJECTED, "tampered metadata refused"); + + ASSERT_EQ(try_load(&f, sig, sig_sz, KEY_SPEC_SESSION_KEYRING, NULL, 0), + -EPERM, "caller-supplied keyring refused once .bpf is in use"); + + err = keyctl_ret(KEYCTL_UNLINK, KEY_SPEC_SESSION_KEYRING, serial); + ASSERT_EQ(err, -ENOENT, "keyring writable while the user bits are there"); + + err = keyctl_ret(KEYCTL_SETPERM, serial, BPF_KEYRING_PERM_LOCKED); + if (!ASSERT_OK(err, "drop the user bits on the bpf keyring")) + goto out; + + ASSERT_OK_FD(try_load(&f, sig, sig_sz, KEY_SPEC_BPF_KEYRING, NULL, 0), + "load still verified against the locked keyring"); + ASSERT_EQ(keyctl_ret(KEYCTL_GET_KEYRING_ID, KEY_SPEC_BPF_KEYRING, 0), + -EACCES, "the special id grants no rights of its own"); + + err = keyctl_ret(KEYCTL_UNLINK, KEY_SPEC_SESSION_KEYRING, serial); + ASSERT_EQ(err, -EACCES, "unlink refused"); + err = keyctl_ret(KEYCTL_CLEAR, serial, 0); + ASSERT_EQ(err, -EACCES, "clear refused"); + err = keyctl_ret(KEYCTL_REVOKE, serial, 0); + ASSERT_EQ(err, -EACCES, "revoke refused"); + err = keyctl_ret(KEYCTL_INVALIDATE, serial, 0); + ASSERT_EQ(err, -EACCES, "invalidate refused"); + err = keyctl_ret(KEYCTL_SET_TIMEOUT, serial, 1); + ASSERT_EQ(err, -EACCES, "timeout refused"); + err = keyctl_ret(KEYCTL_SETPERM, serial, BPF_KEYRING_PERM_INITIAL); + ASSERT_EQ(err, -EACCES, "the bits cannot be granted back"); + + ASSERT_EQ(bpf_keyring_lookup(&nr_keys), serial, "keyring still there"); + ASSERT_EQ(nr_keys, 1, "the enrolled key survived"); +out: + if (have_fixture) + gen_loader_fixture_fini(&f); + genkey_dir_fini(bad_dir); + free(buf); + free(bad); + free(sig); +rmdir: + genkey_dir_fini(dir); +} + /* * A signed loader must ignore ctx-supplied map dimensions: the host cannot * resize a signed program's maps via the loader ctx. Drive a one-map program @@ -1973,68 +2168,71 @@ static void signed_module_kfunc_rejected(void) run_setup("cleanup", dir); } +enum subtest_boot { + BOOT_ANY, + BOOT_SEALED, + BOOT_UNSEALED, +}; + +static const struct { + const char *name; + void (*fn)(void); + enum subtest_boot boot; +} subtests[] = { + { "loadtime_no_map", loadtime_no_map, BOOT_SEALED }, + { "loadtime_with_map", loadtime_with_map, BOOT_SEALED }, + { "metadata_match", metadata_match, BOOT_ANY }, + { "signature_enforced", signature_enforced, BOOT_SEALED }, + { "signed_nonexcl_fd_array_rejected", signed_nonexcl_fd_array_rejected, BOOT_SEALED }, + { "signed_unfrozen_fd_array_rejected", signed_unfrozen_fd_array_rejected, BOOT_SEALED }, + { "signed_nonarray_fd_array_rejected", signed_nonarray_fd_array_rejected, BOOT_SEALED }, + { "signed_btf_fd_array_rejected", signed_btf_fd_array_rejected, BOOT_ANY }, + { "signed_module_kfunc_rejected", signed_module_kfunc_rejected, BOOT_SEALED }, + { "signature_failure_logs", signature_failure_logs, BOOT_SEALED }, + { "signature_too_large", signature_too_large, BOOT_ANY }, + { "signature_zero_size", signature_zero_size, BOOT_ANY }, + { "signature_bad_keyring", signature_bad_keyring, BOOT_SEALED }, + { "bpf_keyring_sealed", bpf_keyring_sealed, BOOT_ANY }, + { "mldsa_signed_load", mldsa_signed_load, BOOT_SEALED }, + { "metadata_ctx_max_entries_ignored", metadata_ctx_max_entries_ignored, BOOT_ANY }, + { "metadata_ctx_initial_value_ignored", metadata_ctx_initial_value_ignored, BOOT_ANY }, + { "signature_authenticates_insns", signature_authenticates_insns, BOOT_SEALED }, + { "signature_authenticates_metadata", signature_authenticates_metadata, BOOT_SEALED }, + { "hash_requires_frozen", hash_requires_frozen, BOOT_ANY }, + { "no_update_after_freeze", no_update_after_freeze, BOOT_ANY }, + { "freeze_writable_mmap", freeze_writable_mmap, BOOT_ANY }, + { "no_writable_mmap_frozen", no_writable_mmap_frozen, BOOT_ANY }, + { "map_hash_matches_libbpf", map_hash_matches_libbpf, BOOT_ANY }, + { "map_hash_multi_element", map_hash_multi_element, BOOT_ANY }, + { "map_hash_bad_size", map_hash_bad_size, BOOT_ANY }, + { "map_hash_unsupported_type", map_hash_unsupported_type, BOOT_ANY }, + { "lsm_signature_verdict", lsm_signature_verdict, BOOT_SEALED }, + { "signed_no_fd_array", signed_no_fd_array, BOOT_SEALED }, + { "signed_map_by_fd_rejected", signed_map_by_fd_rejected, BOOT_SEALED }, + { "signed_sparse_fd_array_rejected", signed_sparse_fd_array_rejected, BOOT_SEALED }, + { "bpf_keyring_provisioned", bpf_keyring_provisioned, BOOT_UNSEALED }, +}; + void test_signed_loader(void) { - if (test__start_subtest("loadtime_no_map")) - loadtime_no_map(); - if (test__start_subtest("loadtime_with_map")) - loadtime_with_map(); - if (test__start_subtest("metadata_match")) - metadata_match(); - if (test__start_subtest("signature_enforced")) - signature_enforced(); - if (test__start_subtest("signed_nonexcl_fd_array_rejected")) - signed_nonexcl_fd_array_rejected(); - if (test__start_subtest("signed_unfrozen_fd_array_rejected")) - signed_unfrozen_fd_array_rejected(); - if (test__start_subtest("signed_nonarray_fd_array_rejected")) - signed_nonarray_fd_array_rejected(); - if (test__start_subtest("signed_btf_fd_array_rejected")) - signed_btf_fd_array_rejected(); - if (test__start_subtest("signed_module_kfunc_rejected")) - signed_module_kfunc_rejected(); - if (test__start_subtest("signature_failure_logs")) - signature_failure_logs(); - if (test__start_subtest("signature_too_large")) - signature_too_large(); - if (test__start_subtest("signature_zero_size")) - signature_zero_size(); - if (test__start_subtest("signature_bad_keyring")) - signature_bad_keyring(); - if (test__start_subtest("bpf_keyring_sealed")) - bpf_keyring_sealed(); - if (test__start_subtest("mldsa_signed_load")) - mldsa_signed_load(); - if (test__start_subtest("metadata_ctx_max_entries_ignored")) - metadata_ctx_max_entries_ignored(); - if (test__start_subtest("metadata_ctx_initial_value_ignored")) - metadata_ctx_initial_value_ignored(); - if (test__start_subtest("signature_authenticates_insns")) - signature_authenticates_insns(); - if (test__start_subtest("signature_authenticates_metadata")) - signature_authenticates_metadata(); - if (test__start_subtest("hash_requires_frozen")) - hash_requires_frozen(); - if (test__start_subtest("no_update_after_freeze")) - no_update_after_freeze(); - if (test__start_subtest("freeze_writable_mmap")) - freeze_writable_mmap(); - if (test__start_subtest("no_writable_mmap_frozen")) - no_writable_mmap_frozen(); - if (test__start_subtest("map_hash_matches_libbpf")) - map_hash_matches_libbpf(); - if (test__start_subtest("map_hash_multi_element")) - map_hash_multi_element(); - if (test__start_subtest("map_hash_bad_size")) - map_hash_bad_size(); - if (test__start_subtest("map_hash_unsupported_type")) - map_hash_unsupported_type(); - if (test__start_subtest("lsm_signature_verdict")) - lsm_signature_verdict(); - if (test__start_subtest("signed_no_fd_array")) - signed_no_fd_array(); - if (test__start_subtest("signed_map_by_fd_rejected")) - signed_map_by_fd_rejected(); - if (test__start_subtest("signed_sparse_fd_array_rejected")) - signed_sparse_fd_array_rejected(); + bool unsealed = keyring_unsealed_boot(); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(subtests); i++) { + if (!test__start_subtest(subtests[i].name)) + continue; + if (subtests[i].boot == BOOT_SEALED && unsealed) { + printf("%s:SKIP:needs a boot without bpf.keyring_unsealed=1\n", + subtests[i].name); + test__skip(); + continue; + } + if (subtests[i].boot == BOOT_UNSEALED && !unsealed) { + printf("%s:SKIP:needs bpf.keyring_unsealed=1\n", + subtests[i].name); + test__skip(); + continue; + } + subtests[i].fn(); + } } From 8588fa592985accc22c6757d6397d588fe71752d Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:27 +0200 Subject: [PATCH 65/90] Documentation/bpf: Document the bpf keyring and improve examples Key generation is detailled for RSA and ML-DSA, the load example sets keyring_id to the bpf keyring with the session keyring shown only as the staging variant, and the LSM admission example anchors on the bpf keyring while allowlisting staged serials rather than treating a user keyring as ordinary trust. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-12-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/bpf/signing.rst | 283 +++++++++++++++++++++++++++++----- 1 file changed, 246 insertions(+), 37 deletions(-) diff --git a/Documentation/bpf/signing.rst b/Documentation/bpf/signing.rst index e73eaaebd8b1..27fdad3df96f 100644 --- a/Documentation/bpf/signing.rst +++ b/Documentation/bpf/signing.rst @@ -254,21 +254,24 @@ returned. Only after the program has fully loaded, at the next hook (``security_bpf_prog()``), does ``BPF_SIG_VERIFIED`` carry its full meaning: validly signed *and* fully verified. -A more realistic admission policy than "is it signed at all": accept programs -signed by a system keyring, accept a user-keyring signature only if the -key/keyring it was verified against is on an explicit allowlist, and emit a -tamper-evident record of every decision so that even denied attempts are -auditable. (Illustrative - error checking elided.) +A more realistic admission policy than "is it signed at all": base trust in +the bpf keyring or an explicit allowlist of a caller-supplied staging key/ +keyring, and emit a record of every decision including denied attempts. +(Illustrative - error checking elided.) .. code-block:: c - /* Serials of user keys/keyrings we additionally trust. */ + /* + * Serials of caller-supplied keyrings we are willing to stage. Empty + * on a system that has committed to the bpf keyring, where the kernel + * refuses them anyway. + */ struct { __uint(type, BPF_MAP_TYPE_HASH); __type(key, __s32); /* keyring_serial */ __type(value, __u8); __uint(max_entries, 64); - } trusted_user_keys SEC(".maps"); + } staging_keys SEC(".maps"); /* Audit stream consumed by a userspace logger. */ struct { @@ -291,11 +294,18 @@ auditable. (Illustrative - error checking elided.) if (kernel) return 0; /* trust in-kernel loads */ - if (verdict != BPF_SIG_VERIFIED) + if (verdict != BPF_SIG_VERIFIED) { ret = -EPERM; /* must be validly signed */ - else if (ktype == BPF_SIG_KEYRING_USER && - !bpf_map_lookup_elem(&trusted_user_keys, &serial)) - ret = -EPERM; /* key/keyring not allowlisted */ + } else switch (ktype) { + case BPF_SIG_KEYRING_BPF: + break; + case BPF_SIG_KEYRING_USER: + if (!bpf_map_lookup_elem(&staging_keys, &serial)) + ret = -EPERM; + break; + default: + ret = -EPERM; /* keyring not in policy */ + } d = bpf_ringbuf_reserve(&audit, sizeof(*d), 0); if (d) { @@ -309,6 +319,11 @@ auditable. (Illustrative - error checking elided.) return ret; } +Such a policy is what makes a caller-supplied keyring usable at all on a system +that does not boot with ``bpf.keyring_unsealed=1``: the allowlist bounds which +staged keys count, and the LSM itself has to protect them from being tampered +with. + Observing a verified load: ``security_bpf_prog()`` -------------------------------------------------- @@ -381,8 +396,9 @@ that verdict covered all of its exclusive maps, rejecting any that did not - so a deny-by-default admission policy needs no second enforcement point. Use ``security_bpf_prog()`` to record or finally gate the verified programs once they carry an id. The ``verdict``, ``keyring_type`` and ``keyring_serial`` fields -let a policy distinguish, for example, "verified and signed by a builtin key" -from "verified by a user key". A policy LSM such as IPE could consume the same +let a policy distinguish "verified against the operator's bpf keyring" from +"verified against a keyring the loader supplied itself", which is the +distinction that matters most. A policy LSM such as IPE could consume the same hooks to enforce system policy without writing any BPF, though none implements this today. @@ -390,33 +406,158 @@ Keyrings ======== ``keyring_id`` selects the trusted keyring the PKCS#7 signature is verified -against. The well-known ids ``0`` (builtin), ``VERIFY_USE_SECONDARY_KEYRING`` -and ``VERIFY_USE_PLATFORM_KEYRING`` select the corresponding system keyrings; -any other value is treated as the serial of a user/session key or keyring. -The keyring is looked up first, before the signature bytes are examined, so a -signature naming a non-existent keyring is rejected up front, and a failed -verification aborts the load - so a program that loads successfully with a -signature always has consistent keyring fields recorded. +against. Four values are well-known; anything else is resolved as a caller- +supplied key or keyring, named either by its serial or by one of the other +``KEY_SPEC_*`` ids: + +.. list-table:: + :header-rows: 1 + + * - ``keyring_id`` + - Keyring + * - ``0`` + - builtin trusted keyring + * - ``VERIFY_USE_SECONDARY_KEYRING`` (``1``) + - secondary trusted keyring + * - ``VERIFY_USE_PLATFORM_KEYRING`` (``2``) + - platform keyring + * - ``KEY_SPEC_BPF_KEYRING`` (``-9``) + - the bpf keyring + * - anything else + - a caller-supplied user/session key or keyring + +The keyring is resolved first, before the signature bytes are examined, so a +signature naming a keyring that cannot be used is rejected up front, and a +failed verification aborts the load - a program that loads successfully with +a signature therefore always has consistent keyring fields recorded. + +The bpf keyring +--------------- + +A system keyring needs a kernel rebuild or a vouched-for enrollment to rotate a +key, and grants BPF-signing trust to keys trusted for everything else in the +kernel too. A caller-supplied keyring, at the other extreme, is filled by the +very process that loads the program and so carries no trust of its own. + +The bpf keyring fills that gap and is the trust anchor which a signed BPF +deployment should be built on top of: a keyring named ``.bpf``, selected with +``KEY_SPEC_BPF_KEYRING``, that an operator provisions at boot with a key +scoped to BPF program loading and nothing else in the kernel's trust hierarchy. +It is owned by the operator rather than by the loader, and rotatable across a +reboot without touching the kernel image. It is modelled after the dm-verity +keyring (see ``dm_verity.keyring_unsealed=``) and provisioned the same way: an +initrd runs the ``keyctl`` steps below before handing off to the rootfs. + +Provisioning +~~~~~~~~~~~~ + +The keyring is created during ``late_initcall`` and is **sealed empty** by +default: it carries a reject-all restriction, so no key can ever be added and +``KEY_SPEC_BPF_KEYRING`` fails with ``-ENOKEY`` for the whole boot. + +``bpf.keyring_unsealed=1`` leaves it unrestricted at init so the initrd can +provision it. The keyring is not linked into any process keyring, but the +``KEY_SPEC_BPF_KEYRING`` special key id addresses it directly, so its serial +does not have to be looked up first. Steps would be as follows:: + + keyctl padd asymmetric "" -9 < signing_key.der + keyctl restrict_keyring -9 + +Both steps are required: the keyring is consulted only once it is **non-empty +and restricted**. An unrestricted keyring is ignored even when it holds keys, +so a half-provisioned keyring is inert rather than a weaker trust anchor, and a +load naming it fails with ``-ENOKEY`` and a verifier log. Restricting cannot +be undone. + +More than one key is enrolled by repeating the ``keyctl padd`` step; the +restriction is applied once, after the last of them:: + + for key in /etc/bpf/keys/*.der; do + keyctl padd asymmetric "" -9 < $key + done + + keyctl restrict_keyring -9 + keyctl show -9 + +The restriction bounds what can be added, never what can be taken away. A key +that is already enrolled can still be unlinked, and the keyring cleared or +revoked, by anything running as root. That does not weaken the anchor, since +a keyring left empty is no longer consulted and a load naming it fails with +``-ENOKEY``, but it does take signed loading out until the next boot. Dropping +the user permissions the keyring no longer needs would close that; as a third +step in the initrd:: + + keyctl setperm -9 0x08030000 + +What remains is ``KEY_POS_SEARCH`` for the in-kernel search during verification, +plus ``KEY_USR_VIEW`` and ``KEY_USR_READ`` so the keyring stays visible in +``/proc/keys``. ``KEY_SPEC_BPF_KEYRING`` names the keyring but conveys no rights +of its own - the keyring is not possessed by anyone, so the ``KEY_POS_*`` bits +are only ever reached by the in-kernel search, and userspace is left with what +the ``KEY_USR_*`` bits grant. Once they are dropped, addressing it by the serial +``/proc/keys`` reports gets no further than the special id does. + +Provisioning has to complete before control passes to the rootfs. The keyring +takes any number of keys for as long as it is unrestricted, so it is the +restriction that bounds the enrolled set, not the first enrollment. Before +the initrd hands off control, it must therefore restrict the keyring after +the last enrollment. + +Enforcement +~~~~~~~~~~~ + +``bpf.keyring_unsealed=1`` states that the bpf keyring is *the* trust anchor for +this boot, so it does more than unseal. From the first program load onwards a +caller-supplied user/session keyring is refused with ``-EPERM`` and a verifier +log message, whether or not provisioning ever completed. The system keyrings +stay selectable. + +Enforcement is readable at ``/sys/module/bpf/parameters/keyring_unsealed``, and +read-only there: the flag is ``__ro_after_init`` behind a 0444 parameter. It is +also derived from the boot flag rather than from the keyring's runtime state, +so there is no window early in boot during which a caller-supplied keyring is +still accepted. + +Caller-supplied keyrings are for staging +---------------------------------------- + +A ``keyring_id`` naming a user or session key or keyring is a *staging* +mechanism, not a trust anchor: it is filled by the same userspace that loads the +program, so verifying against it establishes only that the loader signed what it +loaded. Its purpose is to let software installed onto a running system - whose +signing key is not enrolled anywhere yet - run signed until that key reaches the +bpf keyring on the next boot. + +A system that has committed to the bpf keyring refuses this path outright (see +`Enforcement`_). A system that has not can still allow it, but a policy must +never treat ``BPF_SIG_KEYRING_USER`` as equivalent to the bpf or system +keyrings; it should allowlist the specific serials it is willing to stage and +pair that with a BPF LSM policy protecting those keys from tampering, as in +`Enforcement via LSMs`_. + +Recorded fields +--------------- Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect: ``keyring_type`` (``enum bpf_sig_keyring``) Classified purely from ``keyring_id`` whenever the program is signed: ``BPF_SIG_KEYRING_BUILTIN``, ``_SECONDARY``, ``_PLATFORM`` for the system - keyrings, or ``_USER`` for a user/session keyring. It is - ``BPF_SIG_KEYRING_NONE`` for an unsigned program. + keyrings, ``_BPF`` for the bpf keyring, or ``_USER`` for a caller-supplied + user/session keyring. It is ``BPF_SIG_KEYRING_NONE`` for an unsigned + program. ``keyring_serial`` (``s32``) Set **only** on a successful verification, to the serial of the - **user/session key or keyring** that ``keyring_id`` resolved to - the + **caller-supplied key or keyring** that ``keyring_id`` resolved to - the object the signature was verified against, not the individual asymmetric key inside it that matched the signer. Passing ``KEY_SPEC_SESSION_KEYRING``, for example, records the session keyring's - serial. The system keyrings are trusted as a whole and expose no serial - here, so the serial is ``0`` for builtin, secondary and platform - signatures, and ``0`` for unsigned programs. In other words, a non-zero - ``keyring_serial`` is exactly "verified against the user key/keyring with - this serial". + serial. The system keyrings and the bpf keyring are trusted as a whole and + expose no serial here, so the serial is ``0`` for them, and ``0`` for + unsigned programs. A non-zero ``keyring_serial`` is therefore exactly + "verified against the caller-supplied key/keyring with this serial", which + is exactly the case a policy has to scrutinise. .. list-table:: :header-rows: 1 @@ -436,16 +577,51 @@ Two fields are recorded in ``prog->aux->sig`` for an LSM to inspect: * - ``VERIFY_USE_PLATFORM_KEYRING`` - ``BPF_SIG_KEYRING_PLATFORM`` - ``0`` - * - other (a user/session key serial) + * - ``KEY_SPEC_BPF_KEYRING`` + - ``BPF_SIG_KEYRING_BPF`` + - ``0`` + * - other (a caller-supplied key serial) - ``BPF_SIG_KEYRING_USER`` - serial of the resolved key/keyring -Producing a signed object -========================== +Producing and loading a signed object +===================================== + +Generating a signing key +------------------------ + +Signing is algorithm agnostic: the algorithm comes from the X.509 certificate +and the PKCS#7 ``SignerInfo``. Anything the X.509 and PKCS#7 parsers understand +works with no BPF-side change. Both examples below read the certificate request +config from ``x509.genkey``, for which ``certs/x509.genkey`` serves as a +template (see Documentation/admin-guide/module-signing.rst). RSA:: + + openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 \ + -config x509.genkey -outform PEM \ + -out signing_key.pem -keyout signing_key.pem + openssl x509 -in signing_key.pem -outform der -out signing_key.der + +ML-DSA-87 (FIPS-204), which needs openssl 3.5 or later, and both +``CONFIG_CRYPTO_MLDSA`` and ``CONFIG_CRYPTO_SHA512`` in the kernel - the latter +for the signedAttrs digest described below, which ``CONFIG_CRYPTO_MLDSA`` does +not select. Note the absence of a digest option: ML-DSA hashes the message +itself, so openssl ignores an explicit digest for it:: + + openssl req -new -nodes -utf8 -days 36500 -batch -x509 \ + -newkey ML-DSA-87 -config x509.genkey -outform PEM \ + -out signing_key.pem -keyout signing_key.pem + openssl x509 -in signing_key.pem -outform der -out signing_key.der + +``bpftool`` handles the following internally: openssl before 4.0 cannot combine +ML-DSA with ``CMS_NOATTR``, so it falls back to signedAttrs, where only SHA-512 +is permitted. This mirrors what module signing does as well. + +Signing +------- ``bpftool`` generates and signs a light skeleton in one step:: - bpftool gen skeleton -L -S -k -i \ + bpftool gen skeleton -L -S -k signing_key.pem -i signing_key.der \ obj.bpf.o > obj.lskel.h ``-L`` selects the light-skeleton (``gen_loader``) backend and ``-S`` enables @@ -454,12 +630,36 @@ signing; ``-k`` and ``-i`` supply the signing key and its X.509 certificate. reconstructs - and also computes ``excl_prog_hash`` as the digest of the loader instructions so the metadata map can be bound to the loader. The signature and hash are embedded in the generated header; the certificate is used only for -signing and is not included. Loading the skeleton performs the -create/populate/freeze/load sequence described above. +signing and is not included. -At runtime the trusted public key must be present in the chosen keyring (for -example added to the session keyring, or built into the kernel's builtin trusted -keyring) for verification to succeed. +Loading +------- + +The generated skeleton exposes ``keyring_id``, which selects the keyring the +kernel verifies against. Set it between open and load; loading then performs +the create/populate/freeze/load sequence described above:: + + struct obj *skel = obj__open(); + + skel->keyring_id = KEY_SPEC_BPF_KEYRING; + err = obj__load(skel); + +For the staging case the same object is loaded against a keyring the caller +populated itself, which only works on a system that has not set +``bpf.keyring_unsealed=1``:: + + /* + * Staging only: this keyring is under the loader's own control and + * carries no trust of its own. See "Caller-supplied keyrings are for + * staging". + */ + key_id = add_key("asymmetric", "", der, der_sz, KEY_SPEC_SESSION_KEYRING); + skel->keyring_id = KEY_SPEC_SESSION_KEYRING; + err = obj__load(skel); + +Either way the trusted public key must already be in the chosen keyring for +verification to succeed. For the bpf keyring that enrollment happens once at +boot, see `Provisioning`_. UAPI reference ============== @@ -487,6 +687,13 @@ UAPI reference The map content is not hashed separately at all - it is covered, as bytes, by the program signature. +Kernel command line: + +``bpf.keyring_unsealed=`` + Set to ``1`` to leave the bpf keyring unsealed for provisioning, and to make + it the only non-system keyring a loader may select for the rest of the boot + (see `The bpf keyring`_). + Notes and limitations ====================== @@ -495,3 +702,5 @@ Notes and limitations exceed it. - The metadata container is a single-element array map, accessed through ``map_direct_value_addr``. +- The bpf keyring needs ``CONFIG_KEYS``; without it there is no bpf keyring + and ``KEY_SPEC_BPF_KEYRING`` never resolves. From bd71904697c64fe88e83322a79f8e6e68d0306d1 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:02 -0700 Subject: [PATCH 66/90] selftests/bpf: Add compare_text_to_expected() helper Some selftests generate text (such as BTF dump) and check it against an expectation committed nearby. There is no shared way to do that. prog_tests/btf_dump.c assembles an "awk ... | diff -u" pipeline and hands it to system(). Any other test wanting the same behaviour needs to reproduce both the comparison and the reporting of a mismatch. test_progs captures per-subtest output by pointing the stdout and stderr FILE * globals at a memstream. But a child process inherits descriptors, not the globals. So the "| diff -u" goes to the console instead of the subtest log and is absent from the failure report. Add a helper that compares two strings and runs diff(1) on mismatch, properly relaying the output to stdout. Add tests for the helper. Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-2-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- .../bpf/prog_tests/prog_tests_framework.c | 23 ++++++++++ tools/testing/selftests/bpf/testing_helpers.c | 43 +++++++++++++++++++ tools/testing/selftests/bpf/testing_helpers.h | 3 ++ 3 files changed, 69 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c index 7607cfc2408c..d111fe105447 100644 --- a/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c +++ b/tools/testing/selftests/bpf/prog_tests/prog_tests_framework.c @@ -179,3 +179,26 @@ void test_prog_tests_framework_expected_msgs(void) } } } + +void test_prog_tests_framework_compare_text(void) +{ + int err; + + if (test__start_subtest("compare_text_match")) { + err = compare_text_to_expected("same\n", "same\n"); + ASSERT_EQ(err, 0, "match_rc"); + test__end_subtest(); + } + + if (test__start_subtest("compare_text_mismatch")) { + err = compare_text_to_expected("line two\n", "line one\n"); + fflush(stdout); + + ASSERT_EQ(err, -1, "mismatch_rc"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "-line one", + "diff_has_expected"); + ASSERT_HAS_SUBSTR(env.subtest_state->log_buf, "+line two", + "diff_has_actual"); + test__end_subtest(); + } +} diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index c970e7793dfc..3f037949e978 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -534,3 +534,46 @@ int stack_mprotect(void) PROT_READ | PROT_WRITE | PROT_EXEC); return ret; } + +int compare_text_to_expected(const char *actual, const char *expected) +{ + char exp_path[] = "/tmp/selftest_expected.XXXXXX"; + char act_path[] = "/tmp/selftest_actual.XXXXXX"; + char buf[512], cmd[128]; + int exp_fd, act_fd; + FILE *p; + + if (!strcmp(actual, expected)) + return 0; + + exp_fd = mkstemp(exp_path); + act_fd = mkstemp(act_path); + if (exp_fd < 0 || act_fd < 0) { + fprintf(stdout, "output differs, no temp file for a diff\n"); + goto out; + } + + dprintf(exp_fd, "%s", expected); + dprintf(act_fd, "%s", actual); + + snprintf(cmd, sizeof(cmd), "diff -u '%s' '%s'", exp_path, act_path); + p = popen(cmd, "r"); + if (!p) { + fprintf(stdout, "output differs, '%s' did not run\n", cmd); + goto out; + } + while (fgets(buf, sizeof(buf), p)) + fputs(buf, stdout); + pclose(p); + +out: + if (exp_fd >= 0) { + close(exp_fd); + unlink(exp_path); + } + if (act_fd >= 0) { + close(act_fd); + unlink(act_path); + } + return -1; +} diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index 2edc6fb7fc52..1c58a2f08b64 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -61,4 +61,7 @@ int testing_prog_flags(void); bool is_jit_enabled(void); int stack_mprotect(void); +/* Runs diff(1) on mismatch */ +int compare_text_to_expected(const char *actual, const char *expected); + #endif /* __TESTING_HELPERS_H */ From c0722dc3288aa5f53e058c91bebd100bcc6682d0 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:03 -0700 Subject: [PATCH 67/90] selftests/bpf: Modernize btf_dump test scaffolding Refactor test_btf_dump_case() in order to: * use newer ASSERT_* macros instead of CHECK * drop the file-scope "duration" variable CHECK required Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-3-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/btf_dump.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index 9f1b50e07a29..e9d2d4c26509 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -2,8 +2,6 @@ #include #include -static int duration = 0; - void btf_dump_printf(void *ctx, const char *fmt, va_list args) { vfprintf(ctx, fmt, args); @@ -69,8 +67,9 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) if (!t->known_ptr_sz) { btf__set_pointer_size(btf, 8); } else { - CHECK(btf__pointer_size(btf) != 8, "ptr_sz", "exp %d, got %zu\n", - 8, btf__pointer_size(btf)); + size_t ptr_sz = btf__pointer_size(btf); + + ASSERT_EQ(ptr_sz, (size_t)8, "ptr_sz"); } snprintf(out_file, sizeof(out_file), "/tmp/%s.output.XXXXXX", t->file); @@ -80,8 +79,7 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) goto done; } f = fdopen(fd, "w"); - if (CHECK(f == NULL, "open_tmp", "failed to open file: %s(%d)\n", - strerror(errno), errno)) { + if (!ASSERT_OK_PTR(f, "open_tmp")) { close(fd); goto done; } @@ -89,9 +87,8 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) err = btf_dump_all_types(btf, f); fclose(f); close(fd); - if (CHECK(err, "btf_dump", "failure during C dumping: %d\n", err)) { + if (!ASSERT_OK(err, "btf_dump")) goto done; - } snprintf(test_file, sizeof(test_file), "progs/%s.c", t->file); if (access(test_file, R_OK) == -1) @@ -114,10 +111,10 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) "out {sub(/^[ \\t]*\\*/, \"\"); print}' '%s' | diff -u - '%s'", test_file, out_file); err = system(diff_cmd); - if (CHECK(err, "diff", - "differing test output, output=%s, err=%d, diff cmd:\n%s\n", - out_file, err, diff_cmd)) + if (!ASSERT_OK(err, "diff")) { + fprintf(stdout, "output=%s, diff cmd:\n%s\n", out_file, diff_cmd); goto done; + } remove(out_file); From 122255fb1933a57d81d05d0224d417969ec35fa9 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:04 -0700 Subject: [PATCH 68/90] selftests/bpf: Compare btf_dump expected output in-process test_btf_dump_case() writes the dumped types to a temporary file and then runs a command against the expectation: system("awk '...' 'progs/x.c' | diff -u - '/tmp/x.output.XXXXXX'"); This creates three processes per test case: a shell, awk and diff. The awk cannot be deferred to the failure path, because its output is the expected text. In a VM with a 9p root (used on BPF CI) process creation dominates the cost. A bare fork+exec measures around 110ms, so the btf_dump test cases spend most of their runtime forking. Replace the awk pipeline with a simple marker parser. Buffer the dump in memory, and use compare_text_to_expected(), which only runs diff(1) on mismatch. Measured with "time ./test_progs -t btf_dump" in the VM: real 2.751s -> 0.702s, sys 1.901s -> 0.193s This also drops the temporary file setup. Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-4-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- .../selftests/bpf/prog_tests/btf_dump.c | 112 +++++++++++++----- 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index e9d2d4c26509..fe04a955d46c 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include "testing_helpers.h" void btf_dump_printf(void *ctx, const char *fmt, va_list args) { @@ -43,12 +44,75 @@ static int btf_dump_all_types(const struct btf *btf, void *ctx) return err; } +/* + * Expected output is embedded in the test case source, between + * START-EXPECTED-OUTPUT and END-EXPECTED-OUTPUT markers. A region is either + * plain C, where a declaration doubles as its own expectation, or C wrapped in + * a block comment, where the rendered form differs from the source. All + * regions of a file concatenate into one expectation, compared against one + * whole-file dump. + * + * Returns a malloc'd buffer for the caller to free, or NULL on failure. + */ +static char *read_expected_output(const char *path) +{ + size_t out_sz = 0, line_cap = 0; + char *out = NULL, *line = NULL; + bool in_region = false; + FILE *f, *out_file; + + f = fopen(path, "r"); + if (!f) + return NULL; + + out_file = open_memstream(&out, &out_sz); + if (!out_file) { + fclose(f); + return NULL; + } + + while (getline(&line, &line_cap, f) > 0) { + const char *p; + + if (strstr(line, "START-EXPECTED-OUTPUT")) { + in_region = true; + continue; + } + if (strstr(line, "END-EXPECTED-OUTPUT")) + in_region = false; + if (!in_region) + continue; + + p = line + strspn(line, " \t"); + + /* opening or closing line of a commented out region */ + if (!strncmp(p, "/*", 2) || !strncmp(p, "*/", 2)) + continue; + + /* + * Only a '*' directly after the indentation is a comment + * prefix. Without one the line is taken as it is, leading + * whitespace included. + */ + p = *p == '*' ? p + 1 : line; + + fputs(p, out_file); + } + + free(line); + fclose(f); + fclose(out_file); + return out; +} + static int test_btf_dump_case(int n, struct btf_dump_test_case *t) { - char test_file[256], out_file[256], diff_cmd[1024]; + char *dump = NULL, *expected = NULL; struct btf *btf = NULL; - int err = 0, fd = -1; - FILE *f = NULL; + char test_file[256]; + size_t dump_sz = 0; + int err = 0; + FILE *f; snprintf(test_file, sizeof(test_file), "%s.bpf.o", t->file); @@ -72,21 +136,14 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) ASSERT_EQ(ptr_sz, (size_t)8, "ptr_sz"); } - snprintf(out_file, sizeof(out_file), "/tmp/%s.output.XXXXXX", t->file); - fd = mkstemp(out_file); - if (!ASSERT_GE(fd, 0, "create_tmp")) { - err = fd; - goto done; - } - f = fdopen(fd, "w"); - if (!ASSERT_OK_PTR(f, "open_tmp")) { - close(fd); + f = open_memstream(&dump, &dump_sz); + if (!ASSERT_OK_PTR(f, "open_memstream")) { + err = -errno; goto done; } err = btf_dump_all_types(btf, f); fclose(f); - close(fd); if (!ASSERT_OK(err, "btf_dump")) goto done; @@ -97,28 +154,23 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) * without preserving the directory structure. */ snprintf(test_file, sizeof(test_file), "%s.c", t->file); - /* - * Diff test output and expected test output, contained between - * START-EXPECTED-OUTPUT and END-EXPECTED-OUTPUT lines in test case. - * For expected output lines, everything before '*' is stripped out. - * Also lines containing comment start and comment end markers are - * ignored. - */ - snprintf(diff_cmd, sizeof(diff_cmd), - "awk '/START-EXPECTED-OUTPUT/{out=1;next} " - "/END-EXPECTED-OUTPUT/{out=0} " - "/\\/\\*|\\*\\//{next} " /* ignore comment start/end lines */ - "out {sub(/^[ \\t]*\\*/, \"\"); print}' '%s' | diff -u - '%s'", - test_file, out_file); - err = system(diff_cmd); - if (!ASSERT_OK(err, "diff")) { - fprintf(stdout, "output=%s, diff cmd:\n%s\n", out_file, diff_cmd); + + expected = read_expected_output(test_file); + if (!ASSERT_OK_PTR(expected, "read_expected_output")) { + err = -errno; goto done; } - remove(out_file); + /* + * The mismatch has already been reported, so this only has to + * register the failure. ASSERT_OK() would append a stale errno to it. + */ + err = compare_text_to_expected(dump, expected); + ASSERT_EQ(err, 0, "compare_text_to_expected"); done: + free(expected); + free(dump); btf__free(btf); return err; } From 4f8a4c76f6704667111d76404bdea4fef0373358 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:05 -0700 Subject: [PATCH 69/90] selftests/bpf: NUL-terminate bpftool command output run_command() writes a bpftool command's output into a caller-supplied buffer, and every caller treats that buffer as a C string. However fread() reports a byte count and doesn't terminate the string. The helper does not terminate either, so callers have to zero the buffer first. prog_tests/bpftool_metadata.c does not, for example. Read one byte less and terminate in the helper. Fixes: f21fae577446 ("selftests/bpf: Add a few helpers for bpftool testing") Acked-by: Eduard Zingerman Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-5-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/bpftool_helpers.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/bpftool_helpers.c b/tools/testing/selftests/bpf/bpftool_helpers.c index 0a2a4f0a2794..c49fdd90eb03 100644 --- a/tools/testing/selftests/bpf/bpftool_helpers.c +++ b/tools/testing/selftests/bpf/bpftool_helpers.c @@ -53,6 +53,7 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) static char bpftool_path[PATH_MAX] = {}; bool suppress_output = !(output_buf && output_max_len); char command[BPFTOOL_FULL_CMD_MAX_LEN]; + size_t n; FILE *f; int ret; @@ -68,8 +69,10 @@ static int run_command(char *args, char *output_buf, size_t output_max_len) if (!f) return 1; - if (!suppress_output) - fread(output_buf, 1, output_max_len, f); + if (!suppress_output) { + n = fread(output_buf, 1, output_max_len - 1, f); + output_buf[n] = '\0'; + } ret = pclose(f); return ret; From 438147f7f60104a228fb506352bc8bbae53b1efd Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:06 -0700 Subject: [PATCH 70/90] selftests/bpf: Add tests for bpftool btf dump format c "bpftool btf dump format c" generates the vmlinux.h that BPF programs are built against, and it has no test coverage at all. The only in-tree consumers are build systems. Building against the header only catches what a compiler rejects, which is insufficient. Introduce a bpftool_btf_dump selftest. Dump a small hand-built BTF and compare the output against committed expectations. The BTF is assembled with btf__add_*() rather than compiled from BPF C because the fixture needs a 4-byte "long int". That is what makes bpftool render the hole in struct holey as a pair of "long: 32;" bitfields. A BPF target is always 64-bit, so a compiled fixture could not supply it. Comparing the whole dump makes every change to a generated header show up in a patch. Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-6-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/Makefile | 8 +- .../bpf/bpftool_btf_dump_sorted.expected | 44 +++++ .../bpf/bpftool_btf_dump_unsorted.expected | 48 +++++ .../bpf/prog_tests/bpftool_btf_dump.c | 178 ++++++++++++++++++ 4 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected create mode 100644 tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected create mode 100644 tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 05b3ee64290f..411a13681155 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -119,7 +119,9 @@ TEST_GEN_PROGS += test_progs-cpuv4 TEST_INST_SUBDIRS += cpuv4 endif -TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) +TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ @@ -887,7 +889,9 @@ TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ ima_setup.sh \ $(VERIFY_SIG_SETUP) \ $(wildcard progs/btf_dump_test_case_*.c) \ - $(wildcard progs/*.bpf.o) + $(wildcard progs/*.bpf.o) \ + bpftool_btf_dump_sorted.expected \ + bpftool_btf_dump_unsorted.expected TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS $(eval $(call DEFINE_TEST_RUNNER,test_progs)) diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected new file mode 100644 index 000000000000..a9c6b688ffd7 --- /dev/null +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected @@ -0,0 +1,44 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record) +#endif + +#ifndef __ksym +#define __ksym __attribute__((section(".ksyms"))) +#endif + +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +#ifndef __bpf_fastcall +#if __has_attribute(bpf_fastcall) +#define __bpf_fastcall __attribute__((bpf_fastcall)) +#else +#define __bpf_fastcall +#endif +#endif + +struct holey { + int c; + long: 32; + long: 32; + int tail; +}; + +struct s { + int f; +}; + + +/* BPF kfuncs */ +#ifndef BPF_NO_KFUNC_PROTOTYPES +#endif + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute pop +#endif + +#endif /* __VMLINUX_H__ */ diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected new file mode 100644 index 000000000000..4310adc2ed51 --- /dev/null +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_unsorted.expected @@ -0,0 +1,48 @@ +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute push (__attribute__((preserve_access_index)), apply_to = record) +#endif + +#ifndef __ksym +#define __ksym __attribute__((section(".ksyms"))) +#endif + +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +#ifndef __bpf_fastcall +#if __has_attribute(bpf_fastcall) +#define __bpf_fastcall __attribute__((bpf_fastcall)) +#else +#define __bpf_fastcall +#endif +#endif + +struct holey { + int c; + long: 32; + long: 32; + int tail; +}; + +enum { + E0 = 1, +}; + +struct s { + int f; +}; + + +/* BPF kfuncs */ +#ifndef BPF_NO_KFUNC_PROTOTYPES +#endif + +#ifndef BPF_NO_PRESERVE_ACCESS_INDEX +#pragma clang attribute pop +#endif + +#endif /* __VMLINUX_H__ */ diff --git a/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c new file mode 100644 index 000000000000..d5b25302b0c8 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/bpftool_btf_dump.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "btf_helpers.h" +#include "testing_helpers.h" + +#define DUMP_BUF_SZ (16 * 1024) + +#define EXPECTED_SORTED "bpftool_btf_dump_sorted.expected" +#define EXPECTED_UNSORTED "bpftool_btf_dump_unsorted.expected" + +/* + * struct holey { + * int c; + * <64-bit hole> + * int tail; + * }; + * enum { E0 = 1 }; + * struct s { int f; }; + */ +static struct btf *mk_btf(void) +{ + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "new_empty")) + return NULL; + + btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + btf__add_int(btf, "long int", 4, BTF_INT_SIGNED); + + btf__add_struct(btf, "holey", 16); + btf__add_field(btf, "c", 1, 0, 0); + btf__add_field(btf, "tail", 1, 96, 0); + + btf__add_enum(btf, NULL, 4); + btf__add_enum_value(btf, "E0", 1); + + btf__add_struct(btf, "s", 4); + btf__add_field(btf, "f", 1, 0, 0); + + VALIDATE_RAW_BTF( + btf, + "[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[2] INT 'long int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[3] STRUCT 'holey' size=16 vlen=2\n" + "\t'c' type_id=1 bits_offset=0\n" + "\t'tail' type_id=1 bits_offset=96", + "[4] ENUM '(anon)' encoding=UNSIGNED size=4 vlen=1\n" + "\t'E0' val=1", + "[5] STRUCT 's' size=4 vlen=1\n" + "\t'f' type_id=1 bits_offset=0"); + + return btf; +} + +static int btf_to_tmpfile(const struct btf *btf, char *path) +{ + ssize_t written; + const void *raw; + __u32 sz; + int fd; + + raw = btf__raw_data(btf, &sz); + if (!ASSERT_OK_PTR(raw, "raw_data")) + return -1; + + snprintf(path, PATH_MAX, "/tmp/bpftool_btf_dump.XXXXXX"); + fd = mkstemp(path); + if (!ASSERT_OK_FD(fd, "mkstemp_btf")) + return -1; + + written = write(fd, raw, sz); + close(fd); + if (!ASSERT_EQ(written, sz, "write_btf")) { + unlink(path); + return -1; + } + + return 0; +} + +static char *dump_c(const char *btf_path, bool sorted) +{ + char args[MAX_BPFTOOL_CMD_LEN]; + char *buf; + int err; + + buf = malloc(DUMP_BUF_SZ); + if (!ASSERT_OK_PTR(buf, "alloc_dump")) + return NULL; + + snprintf(args, sizeof(args), "btf dump file %s format c%s", + btf_path, sorted ? "" : " unsorted"); + + err = get_bpftool_command_output(args, buf, DUMP_BUF_SZ); + if (!ASSERT_OK(err, "btf_dump_format_c")) { + free(buf); + return NULL; + } + + return buf; +} + +static char *read_expected(const char *path) +{ + char *buf = NULL; + size_t cap = 0; + FILE *f; + int err; + + f = fopen(path, "r"); + if (!f) { + err = errno; + PRINT_FAIL("can't open expected output '%s': errno %d\n", path, err); + return NULL; + } + + /* no NUL in a generated header, so this reads to the end */ + if (getdelim(&buf, &cap, '\0', f) < 0) { + err = errno; + PRINT_FAIL("can't read expected output '%s': errno %d\n", path, err); + free(buf); + buf = NULL; + } + + fclose(f); + return buf; +} + +static void test_dump(const char *btf_path, bool sorted) +{ + const char *exp_path; + char *dump, *exp; + int err; + + exp_path = sorted ? EXPECTED_SORTED : EXPECTED_UNSORTED; + + dump = dump_c(btf_path, sorted); + if (!dump) + return; + + exp = read_expected(exp_path); + if (!exp) + goto out_dump; + + err = compare_text_to_expected(dump, exp); + ASSERT_OK(err, sorted ? "cmp_sorted" : "cmp_unsorted"); + + free(exp); +out_dump: + free(dump); +} + +void test_bpftool_btf_dump(void) +{ + char path[PATH_MAX]; + struct btf *btf; + + btf = mk_btf(); + if (!btf) + return; + + if (btf_to_tmpfile(btf, path)) + goto out_btf; + + if (test__start_subtest("c_sorted")) + test_dump(path, true); + if (test__start_subtest("c_unsorted")) + test_dump(path, false); + + unlink(path); +out_btf: + btf__free(btf); +} From ff01f88a91368c264efb071c13f2b27b4d5c1d2b Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Fri, 28 Aug 2026 14:52:07 -0700 Subject: [PATCH 71/90] bpftool: Don't drop a type in the sorted C dump The C dump sorts types by default, so that generated headers are diffable. The sorted dump emits one type fewer than the unsorted dump of the same BTF. dump_btf_c() starts its loop at index 1 to skip the void type at BTF type ID 0. That holds for the unsorted dump, where the array index is the type ID, but not after qsort(): position 0 is then the lowest ranked type, and btf_type_rank() ranks an anonymous enum 0 while void takes the default rank of 10. So the enum is skipped, and void is emitted instead as a no-op. Fixes: 94133cf24bb3 ("bpftool: Introduce btf c dump sorting") Acked-by: Eduard Zingerman Signed-off-by: Ihor Solodrai Link: https://lore.kernel.org/r/20260828215207.3105313-7-ihor.solodrai@linux.dev Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/btf.c | 6 +++++- .../testing/selftests/bpf/bpftool_btf_dump_sorted.expected | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index c9589026da8d..bca0a3982f09 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -805,9 +805,13 @@ static int dump_btf_c(const struct btf *btf, if (sort_dump) datums = sort_btf_c(btf); - for (i = 1; i < cnt; i++) { + for (i = 0; i < cnt; i++) { int idx = datums ? datums[i].index : i; + /* type ID 0 is void, skip it */ + if (idx == 0) + continue; + err = btf_dump__dump_type(d, idx); if (err) goto done; diff --git a/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected index a9c6b688ffd7..5470b2b43229 100644 --- a/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected +++ b/tools/testing/selftests/bpf/bpftool_btf_dump_sorted.expected @@ -21,6 +21,10 @@ #endif #endif +enum { + E0 = 1, +}; + struct holey { int c; long: 32; From d761934c9483ecde93fe99d8705282f716dfee50 Mon Sep 17 00:00:00 2001 From: Sean Young Date: Thu, 20 Aug 2026 22:43:29 +0100 Subject: [PATCH 72/90] selftests/bpf: Convert lirc_mode2 to prog_tests and extend coverage While porting, extend coverage of the BPF_LIRC_MODE2 attach/detach/ query API: - bpf_prog_attach() with invalid flags is rejected with -EINVAL and does not attach the program - bpf_prog_query() with invalid flags is rejected with -EINVAL without disturbing existing attachments - bpf_prog_query() reports the correct program id, not just count, at each step, via bpf_prog_get_info_by_fd() - a lirc chardev can hold more than one attached program: load a second, independent instance, attach it alongside the first, confirm both are reported by bpf_prog_query(), then detach it without disturbing the first program's attachment - detaching an already-detached program consistently fails with -ENOENT, for both the first and second program Signed-off-by: Sean Young Assisted-by: Claude:claude-sonnet-5 Link: https://lore.kernel.org/bpf/20260820214332.89030-1-sean@mess.org Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/.gitignore | 1 - tools/testing/selftests/bpf/Makefile | 3 - .../selftests/bpf/prog_tests/lirc_mode2.c | 334 ++++++++++++++++++ .../testing/selftests/bpf/progs/lirc_mode2.c | 32 ++ .../bpf/progs/test_lirc_mode2_kern.c | 26 -- .../testing/selftests/bpf/test_lirc_mode2.sh | 41 --- .../selftests/bpf/test_lirc_mode2_user.c | 177 ---------- 7 files changed, 366 insertions(+), 248 deletions(-) create mode 100644 tools/testing/selftests/bpf/prog_tests/lirc_mode2.c create mode 100644 tools/testing/selftests/bpf/progs/lirc_mode2.c delete mode 100644 tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c delete mode 100755 tools/testing/selftests/bpf/test_lirc_mode2.sh delete mode 100644 tools/testing/selftests/bpf/test_lirc_mode2_user.c diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore index 986a6389186b..b815bf0d8877 100644 --- a/tools/testing/selftests/bpf/.gitignore +++ b/tools/testing/selftests/bpf/.gitignore @@ -17,7 +17,6 @@ test_verifier_log feature urandom_read test_sockmap -test_lirc_mode2_user flow_dissector_load test_tcpnotify_user test_libbpf diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 411a13681155..b481b867f372 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -125,7 +125,6 @@ TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) \ # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ - test_lirc_mode2.sh \ test_bpftool_build.sh \ test_doc_build.sh \ test_xsk.sh \ @@ -143,7 +142,6 @@ TEST_GEN_PROGS_EXTENDED = \ bench \ flow_dissector_load \ test_cpp \ - test_lirc_mode2_user \ veristat \ xdp_features \ xdp_hw_metadata \ @@ -340,7 +338,6 @@ $(OUTPUT)/test_sockmap: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELPERS) $(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(OUTPUT)/test_tag: $(TESTING_HELPERS) -$(OUTPUT)/test_lirc_mode2_user: $(TESTING_HELPERS) $(OUTPUT)/flow_dissector_load: $(TESTING_HELPERS) $(OUTPUT)/test_maps: $(TESTING_HELPERS) $(OUTPUT)/test_verifier: $(TESTING_HELPERS) $(CAP_HELPERS) $(UNPRIV_HELPERS) diff --git a/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c b/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c new file mode 100644 index 000000000000..7b7352dbcd82 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/lirc_mode2.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +// test ir decoder +// +// Copyright (C) 2018 Sean Young + +// A lirc chardev is a device representing a consumer IR (cir) device which +// can receive infrared signals from remote control and/or transmit IR. +// +// IR is sent as a series of pulses and space somewhat like morse code. The +// BPF program can decode this into scancodes so that rc-core can translate +// this into input key codes using the rc keymap. +// +// This test works by sending IR over rc-loopback, so the IR is processed by +// BPF and then decoded into scancodes. The lirc chardev must be the one +// associated with rc-loopback, see the output of ir-keytable(1). +// +// The following CONFIG options must be enabled for the test to succeed: +// CONFIG_RC_CORE=y +// CONFIG_BPF_LIRC_MODE2=y +// CONFIG_RC_LOOPBACK=y +// CONFIG_LIRC=y + +#include +#include +#include +#include +#include +#include +#include "lirc_mode2.skel.h" + +/* + * Read the DEVNAME= line out of the first uevent file that matches + * pattern, and turn it into a /dev/ path. + */ +static bool find_devname(const char *pattern, char *path, size_t path_sz) +{ + glob_t gl = {}; + bool found = false; + FILE *f; + + if (glob(pattern, 0, NULL, &gl) || gl.gl_pathc == 0) + goto out; + + f = fopen(gl.gl_pathv[0], "r"); + if (!f) + goto out; + + char line[256]; + + while (fgets(line, sizeof(line), f)) { + char *val; + + if (strncmp(line, "DEVNAME=", 8)) + continue; + + val = line + 8; + val[strcspn(val, "\n")] = '\0'; + snprintf(path, path_sz, "/dev/%s", val); + found = true; + break; + } + + fclose(f); +out: + globfree(&gl); + return found; +} + +/* Load rc-loopback and find the lirc and input chardevs it created. */ +static bool find_loopback_devices(char *lirc_path, char *input_path, + size_t path_sz) +{ + glob_t gl = {}; + bool found = false; + + /* Ignore failure, we check for the resulting devices below. */ + SYS_NOFAIL("modprobe rc-loopback > /dev/null 2>&1"); + + if (glob("/sys/class/rc/rc*", 0, NULL, &gl)) + goto out; + + for (size_t i = 0; i < gl.gl_pathc; i++) { + const char *rcdir = gl.gl_pathv[i]; + char uevent_path[PATH_MAX]; + char uevent[4096]; + char pattern[PATH_MAX]; + FILE *f; + size_t n; + + snprintf(uevent_path, sizeof(uevent_path), "%s/uevent", rcdir); + f = fopen(uevent_path, "r"); + if (!f) + continue; + n = fread(uevent, 1, sizeof(uevent) - 1, f); + fclose(f); + uevent[n] = '\0'; + + if (!strstr(uevent, "DRV_NAME=rc-loopback")) + continue; + + snprintf(pattern, sizeof(pattern), "%s/lirc*/uevent", rcdir); + if (!find_devname(pattern, lirc_path, path_sz)) + continue; + + snprintf(pattern, sizeof(pattern), "%s/input*/event*/uevent", rcdir); + if (!find_devname(pattern, input_path, path_sz)) + continue; + + found = true; + break; + } + +out: + if (!found) { + fprintf(stderr, "No rc devices found\n"); + fprintf(stderr, "Enable CONFIG_RC_LOOPBACK and CONFIG_BPF_LIRC_MODE2\n"); + } + + globfree(&gl); + return found; +} + +void test_lirc_mode2(void) +{ + char lirc_path[PATH_MAX], input_path[PATH_MAX]; + int lircfd = -1, inputfd = -1, progfd, progfd2 = -1; + struct lirc_mode2 *skel = NULL, *skel2 = NULL; + __u32 prog_ids[10], prog_flags[10], prog_cnt; + struct bpf_prog_info info; + __u32 info_len, prog_id, prog_id2; + int testir1 = 0x8ead; /* keydown flag (0x8000) | scancode 0xead */ + int testir2 = 0x4081; /* pointer_rel flag (0x4000) | rel_x=1 | rel_y=1 */ + struct input_event event; + struct pollfd pfd = {}; + int ret; + + if (getuid() != 0) { + test__skip(); + return; + } + + if (!find_loopback_devices(lirc_path, input_path, sizeof(lirc_path))) { + test__skip(); + return; + } + + skel = lirc_mode2__open_and_load(); + if (!ASSERT_OK_PTR(skel, "lirc_mode2__open_and_load")) + return; + + progfd = bpf_program__fd(skel->progs.bpf_decoder); + + memset(&info, 0, sizeof(info)); + info_len = sizeof(info); + ret = bpf_prog_get_info_by_fd(progfd, &info, &info_len); + if (!ASSERT_OK(ret, "get first program's info")) + goto out; + prog_id = info.id; + + lircfd = open(lirc_path, O_RDWR | O_NONBLOCK); + if (!ASSERT_GE(lircfd, 0, "open lirc device")) + goto out; + + /* Try to detach it before it was ever attached, should fail. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + if (!ASSERT_EQ(ret, -ENOENT, "detach unattached program")) + goto out; + + inputfd = open(input_path, O_RDONLY | O_NONBLOCK); + if (!ASSERT_GE(inputfd, 0, "open input device")) + goto out; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs before attach")) + goto out; + if (!ASSERT_EQ(prog_cnt, 0, "no programs should be attached yet")) + goto out; + + /* Invalid attach flags must be rejected, and must not attach. */ + ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 1); + if (!ASSERT_EQ(ret, -EINVAL, "attach with invalid flags")) + goto out; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after rejected attach")) + goto out; + if (!ASSERT_EQ(prog_cnt, 0, "rejected attach should not attach")) + goto out; + + ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 0); + if (!ASSERT_OK(ret, "attach program to lirc device")) + goto out; + + /* Invalid query flags must be rejected too, without upsetting state. */ + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 1, prog_flags, prog_ids, + &prog_cnt); + ASSERT_EQ(ret, -EINVAL, "query with invalid flags"); + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after attach")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should be attached")) + goto out_detach; + ASSERT_EQ(prog_ids[0], prog_id, "queried id should match attached program"); + + /* Write raw IR */ + ret = write(lircfd, &testir1, sizeof(testir1)); + if (!ASSERT_EQ(ret, sizeof(testir1), "send test IR message 1")) + goto out_detach; + + pfd.fd = inputfd; + pfd.events = POLLIN; + + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (!ASSERT_EQ(ret, sizeof(event), "read decoded IR 1")) + goto out_detach; + + if (event.type == EV_MSC && event.code == MSC_SCAN && + event.value == 0xead) + break; + } + + /* Write raw IR */ + ret = write(lircfd, &testir2, sizeof(testir2)); + if (!ASSERT_EQ(ret, sizeof(testir2), "send test IR message 2")) + goto out_detach; + + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (!ASSERT_EQ(ret, sizeof(event), "read decoded IR 2")) + goto out_detach; + + if (event.type == EV_REL && event.code == REL_Y && + event.value == 1) + break; + } + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after IR was decoded")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should still be attached")) + goto out_detach; + + /* + * The lirc chardev can hold more than one attached program at once. + * Load a second, independent instance and check it can be attached + * alongside the first, queried, and then detached on its own + * without disturbing the first program's attachment. + */ + skel2 = lirc_mode2__open_and_load(); + if (!ASSERT_OK_PTR(skel2, "lirc_mode2__open_and_load (2nd)")) + goto out_detach; + + progfd2 = bpf_program__fd(skel2->progs.bpf_decoder); + + memset(&info, 0, sizeof(info)); + info_len = sizeof(info); + ret = bpf_prog_get_info_by_fd(progfd2, &info, &info_len); + if (!ASSERT_OK(ret, "get second program's info")) + goto out_detach; + prog_id2 = info.id; + + ret = bpf_prog_attach(progfd2, lircfd, BPF_LIRC_MODE2, 0); + if (!ASSERT_OK(ret, "attach second program to lirc device")) + goto out_detach; + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after second attach")) + goto out_detach2; + if (!ASSERT_EQ(prog_cnt, 2, "two programs should be attached")) + goto out_detach2; + ASSERT_TRUE((prog_ids[0] == prog_id && prog_ids[1] == prog_id2) || + (prog_ids[0] == prog_id2 && prog_ids[1] == prog_id), + "queried ids should be the two attached programs"); + + /* Detach the second program; the first should remain attached. */ + ret = bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + if (!ASSERT_OK(ret, "detach second program")) + goto out_detach2; + + /* Detaching an already-detached program should now fail. */ + ret = bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + ASSERT_EQ(ret, -ENOENT, "detach second program again"); + + prog_cnt = ARRAY_SIZE(prog_ids); + ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, + &prog_cnt); + if (!ASSERT_OK(ret, "query programs after second detach")) + goto out_detach; + if (!ASSERT_EQ(prog_cnt, 1, "one program should remain attached")) + goto out_detach; + ASSERT_EQ(prog_ids[0], prog_id, "remaining program should be the first one"); + +out_detach: + /* Let's try detaching it now it is actually attached. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + ASSERT_OK(ret, "detach program from lirc device"); + + /* Detaching it again should now fail the same way. */ + ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); + ASSERT_EQ(ret, -ENOENT, "detach program from lirc device again"); + goto out; + +out_detach2: + /* Best-effort cleanup of the second program before bailing out. */ + bpf_prog_detach2(progfd2, lircfd, BPF_LIRC_MODE2); + goto out_detach; + +out: + if (inputfd >= 0) + close(inputfd); + if (lircfd >= 0) + close(lircfd); + lirc_mode2__destroy(skel2); + lirc_mode2__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/lirc_mode2.c b/tools/testing/selftests/bpf/progs/lirc_mode2.c new file mode 100644 index 000000000000..98137f3c5c03 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/lirc_mode2.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +// test ir decoder +// +// Copyright (C) 2018 Sean Young + +#include +#include +#include + +SEC("lirc_mode2") +int bpf_decoder(unsigned int *sample) +{ + if (LIRC_IS_PULSE(*sample)) { + unsigned int duration = LIRC_VALUE(*sample); + + /* + * Flag bits picked deliberately low: rc-loopback simulates + * a receiver overflow for any pulse over MS_TO_US(50) (see + * loop_tx_ir() in rc-loopback.c), which would silently + * swallow the sample before it ever reaches this decoder. + */ + if (duration & 0x8000) + bpf_rc_keydown(sample, 0x40, duration & 0x3fff, 0); + if (duration & 0x4000) + bpf_rc_pointer_rel(sample, (duration >> 7) & 0x7f, + duration & 0x7f); + } + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c b/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c deleted file mode 100644 index cbe4284c032f..000000000000 --- a/tools/testing/selftests/bpf/progs/test_lirc_mode2_kern.c +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// test ir decoder -// -// Copyright (C) 2018 Sean Young - -#include -#include -#include - -SEC("lirc_mode2") -int bpf_decoder(unsigned int *sample) -{ - if (LIRC_IS_PULSE(*sample)) { - unsigned int duration = LIRC_VALUE(*sample); - - if (duration & 0x1000) - bpf_rc_keydown(sample, 0x40, duration & 0xffff, 0); - if (duration & 0x2000) - bpf_rc_pointer_rel(sample, (duration >> 8) & 0xff, - duration & 0xff); - } - - return 0; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_lirc_mode2.sh b/tools/testing/selftests/bpf/test_lirc_mode2.sh deleted file mode 100755 index 5252b91f48a1..000000000000 --- a/tools/testing/selftests/bpf/test_lirc_mode2.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 - -# Kselftest framework requirement - SKIP code is 4. -ksft_skip=4 -ret=$ksft_skip - -msg="skip all tests:" -if [ $UID != 0 ]; then - echo $msg please run this as root >&2 - exit $ksft_skip -fi - -GREEN='\033[0;92m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -modprobe rc-loopback - -for i in /sys/class/rc/rc* -do - if grep -q DRV_NAME=rc-loopback $i/uevent - then - LIRCDEV=$(grep DEVNAME= $i/lirc*/uevent | sed sQDEVNAME=Q/dev/Q) - INPUTDEV=$(grep DEVNAME= $i/input*/event*/uevent | sed sQDEVNAME=Q/dev/Q) - fi -done - -if [ -n "$LIRCDEV" ]; -then - TYPE=lirc_mode2 - ./test_lirc_mode2_user $LIRCDEV $INPUTDEV - ret=$? - if [ $ret -ne 0 ]; then - echo -e ${RED}"FAIL: $TYPE"${NC} - else - echo -e ${GREEN}"PASS: $TYPE"${NC} - fi -fi - -exit $ret diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c deleted file mode 100644 index cd191da20d14..000000000000 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// test ir decoder -// -// Copyright (C) 2018 Sean Young - -// A lirc chardev is a device representing a consumer IR (cir) device which -// can receive infrared signals from remote control and/or transmit IR. -// -// IR is sent as a series of pulses and space somewhat like morse code. The -// BPF program can decode this into scancodes so that rc-core can translate -// this into input key codes using the rc keymap. -// -// This test works by sending IR over rc-loopback, so the IR is processed by -// BPF and then decoded into scancodes. The lirc chardev must be the one -// associated with rc-loopback, see the output of ir-keytable(1). -// -// The following CONFIG options must be enabled for the test to succeed: -// CONFIG_RC_CORE=y -// CONFIG_BPF_RAWIR_EVENT=y -// CONFIG_RC_LOOPBACK=y - -// Steps: -// 1. Open the /dev/lircN device for rc-loopback (given on command line) -// 2. Attach bpf_lirc_mode2 program which decodes some IR. -// 3. Send some IR to the same IR device; since it is loopback, this will -// end up in the bpf program -// 4. bpf program should decode IR and report keycode -// 5. We can read keycode from same /dev/lirc device - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "bpf_util.h" -#include -#include - -#include "testing_helpers.h" - -int main(int argc, char **argv) -{ - struct bpf_object *obj; - int ret, lircfd, progfd, inputfd; - int testir1 = 0x1ead; - int testir2 = 0x2101; - u32 prog_ids[10], prog_flags[10], prog_cnt; - - if (argc != 3) { - printf("Usage: %s /dev/lircN /dev/input/eventM\n", argv[0]); - return 2; - } - - ret = bpf_prog_test_load("test_lirc_mode2_kern.bpf.o", - BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd); - if (ret) { - printf("Failed to load bpf program\n"); - return 1; - } - - lircfd = open(argv[1], O_RDWR | O_NONBLOCK); - if (lircfd == -1) { - printf("failed to open lirc device %s: %m\n", argv[1]); - return 1; - } - - /* Let's try detach it before it was ever attached */ - ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret != -ENOENT) { - printf("bpf_prog_detach2 not attached should fail: %m\n"); - return 1; - } - - inputfd = open(argv[2], O_RDONLY | O_NONBLOCK); - if (inputfd == -1) { - printf("failed to open input device %s: %m\n", argv[1]); - return 1; - } - - prog_cnt = 10; - ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, - &prog_cnt); - if (ret) { - printf("Failed to query bpf programs on lirc device: %m\n"); - return 1; - } - - if (prog_cnt != 0) { - printf("Expected nothing to be attached\n"); - return 1; - } - - ret = bpf_prog_attach(progfd, lircfd, BPF_LIRC_MODE2, 0); - if (ret) { - printf("Failed to attach bpf to lirc device: %m\n"); - return 1; - } - - /* Write raw IR */ - ret = write(lircfd, &testir1, sizeof(testir1)); - if (ret != sizeof(testir1)) { - printf("Failed to send test IR message: %m\n"); - return 1; - } - - struct pollfd pfd = { .fd = inputfd, .events = POLLIN }; - struct input_event event; - - for (;;) { - poll(&pfd, 1, 100); - - /* Read decoded IR */ - ret = read(inputfd, &event, sizeof(event)); - if (ret != sizeof(event)) { - printf("Failed to read decoded IR: %m\n"); - return 1; - } - - if (event.type == EV_MSC && event.code == MSC_SCAN && - event.value == 0x1ead) { - break; - } - } - - /* Write raw IR */ - ret = write(lircfd, &testir2, sizeof(testir2)); - if (ret != sizeof(testir2)) { - printf("Failed to send test IR message: %m\n"); - return 1; - } - - for (;;) { - poll(&pfd, 1, 100); - - /* Read decoded IR */ - ret = read(inputfd, &event, sizeof(event)); - if (ret != sizeof(event)) { - printf("Failed to read decoded IR: %m\n"); - return 1; - } - - if (event.type == EV_REL && event.code == REL_Y && - event.value == 1 ) { - break; - } - } - - prog_cnt = 10; - ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, - &prog_cnt); - if (ret) { - printf("Failed to query bpf programs on lirc device: %m\n"); - return 1; - } - - if (prog_cnt != 1) { - printf("Expected one program to be attached\n"); - return 1; - } - - /* Let's try detaching it now it is actually attached */ - ret = bpf_prog_detach2(progfd, lircfd, BPF_LIRC_MODE2); - if (ret) { - printf("bpf_prog_detach2: returned %m\n"); - return 1; - } - - return 0; -} From 802a30c5db726639df7f93b3a1982ecc64202abe Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 31 Aug 2026 14:32:25 +0800 Subject: [PATCH 73/90] bpf: Fix queue/stack map u32 index overflow The queue/stack map addresses elements[] with the product of a u32 head/tail index and value_size, but the storage itself is allocated in 64-bit arithmetic. When max_entries * value_size reaches or exceeds U32_MAX, the product wraps and push/peek/pop operate on the wrong element, corrupting map data and leaking stale values to user space. max_entries == U32_MAX would also make the u32 capacity counter qs->size (max_entries + 1) wrap to 0 and permanently break the map. The original bound check was removed by commit a37fb7ef24a4 ("bpf: Eliminate rlimit-based memory accounting for queue_stack_maps maps"), which deleted the bpf_map_charge_init() call and with it the U32_MAX - PAGE_SIZE check that had earlier been moved into bpf_map_charge_init() by c85d69135a91. Oversized queue/stack maps can therefore be created again. Restore the bound in queue_stack_map_alloc_check() with a single comparison that rejects any max_entries/value_size combination whose element storage would reach or exceed U32_MAX bytes, keeping the u32 index multiplication overflow-free and the capacity counter valid. Fixes: a37fb7ef24a4 ("bpf: Eliminate rlimit-based memory accounting for queue_stack_maps maps") Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Cc: stable@vger.kernel.org Link: https://lore.kernel.org/bpf/20260831063226.621309-2-chenyuan_fl@163.com --- kernel/bpf/queue_stack_maps.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 6e8b18c32a10..1cee2ba6e332 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -59,6 +59,9 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr) */ return -E2BIG; + if (attr->max_entries >= U32_MAX / attr->value_size) + return -E2BIG; + return 0; } From 33576bfee5b7f65e3df651ac33f83fdcf1f04233 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 31 Aug 2026 14:32:26 +0800 Subject: [PATCH 74/90] selftests/bpf: Add regression test for queue/stack map size limit Verify that queue/stack maps whose element storage would overflow the u32 head/tail index multiplication are rejected at creation time, and that max_entries == U32_MAX (which would wrap the u32 capacity counter to 0) is rejected as well. Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260831063226.621309-3-chenyuan_fl@163.com --- .../bpf/prog_tests/queue_stack_map.c | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c index 41441325e179..a01a0bcbdee7 100644 --- a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c +++ b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c @@ -2,6 +2,8 @@ #include #include +#define U32_MAX ((u32)UINT_MAX) + enum { QUEUE, STACK, @@ -101,8 +103,35 @@ static void test_queue_stack_map_by_type(int type) bpf_object__close(obj); } +static void test_queue_stack_map_alloc_check(void) +{ + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 1000000, 8192, NULL); + ASSERT_EQ(fd, -E2BIG, "queue_oversize"); + if (fd >= 0) + close(fd); + + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 1, U32_MAX, NULL); + ASSERT_EQ(fd, -E2BIG, "queue_u32max"); + if (fd >= 0) + close(fd); + + fd = bpf_map_create(BPF_MAP_TYPE_STACK, NULL, 0, 1000000, 8192, NULL); + ASSERT_EQ(fd, -E2BIG, "stack_oversize"); + if (fd >= 0) + close(fd); + + /* A normal-sized map must still be created successfully. */ + fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, 64, 100, NULL); + ASSERT_GE(fd, 0, "queue_normal"); + if (fd >= 0) + close(fd); +} + void test_queue_stack_map(void) { test_queue_stack_map_by_type(QUEUE); test_queue_stack_map_by_type(STACK); + test_queue_stack_map_alloc_check(); } From 06d6f845c4bda857a6ae864690f35321b878e827 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 1 Sep 2026 10:46:54 +0200 Subject: [PATCH 75/90] bpftool: Set BPF_F_XDP_DEV_BOUND_ONLY flag non-destructively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When setting the XDP hints ifname, bpftool would set the BPF_F_XDP_DEV_BOUND_ONLY without looking at the existing program flags, overriding any other flag values. This was always a destructive action, but after we change libbpf to carry the frags section flag in prog_flags, this can impact bpftool loading of XDP frags programs. Change the flag setting to be non-destructive by OR'ing it with the existing flags. Fixes: f46392ee3dec ("bpftool: Specify XDP Hints ifname when loading program") Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Reviewed-by: Larysa Zaremba Link: https://lore.kernel.org/bpf/20260901-libbpf-frags-flags-v3-1-4eb6f14968b0@redhat.com --- tools/bpf/bpftool/prog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index a9f730d407a9..8c2f9255b36d 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -1769,7 +1769,7 @@ static int load_with_options(int argc, char **argv, bool first_prog_only) } if (prog_type == BPF_PROG_TYPE_XDP && xdpmeta_ifindex) { - bpf_program__set_flags(pos, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__set_flags(pos, bpf_program__flags(pos) | BPF_F_XDP_DEV_BOUND_ONLY); bpf_program__set_ifindex(pos, xdpmeta_ifindex); } else { bpf_program__set_ifindex(pos, offload_ifindex); From d6746a246b362c3c83df170eda6d3c44cc0a4040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 1 Sep 2026 10:46:55 +0200 Subject: [PATCH 76/90] selftests/bpf: Set BPF program flags non-destructively MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A couple of the BPF selftests would set the program flags without looking at the existing program flags, overriding any other flag values. To avoid issues when we change libbpf to carry section flags in prog_flags, change this to always set the flag value non-destructively by OR'ing it with the existing flags. Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260901-libbpf-frags-flags-v3-2-4eb6f14968b0@redhat.com --- tools/testing/selftests/bpf/prog_tests/attach_probe.c | 6 ++++-- tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c | 2 +- tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c | 2 +- tools/testing/selftests/bpf/prog_tests/xdp_metadata.c | 4 ++-- tools/testing/selftests/bpf/xdp_hw_metadata.c | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c index e8c1a619e330..7dadb90e7b68 100644 --- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c +++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c @@ -543,8 +543,10 @@ static void test_kprobe_sleepable(void) return; /* sleepable kprobe test case needs flags set before loading */ - if (!ASSERT_OK(bpf_program__set_flags(skel->progs.handle_kprobe_sleepable, - BPF_F_SLEEPABLE), "kprobe_sleepable_flags")) + if (!ASSERT_OK(bpf_program__set_flags( + skel->progs.handle_kprobe_sleepable, + bpf_program__flags(skel->progs.handle_kprobe_sleepable) | BPF_F_SLEEPABLE), + "kprobe_sleepable_flags")) goto cleanup; if (!ASSERT_OK(test_attach_kprobe_sleepable__load(skel), diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c index 73f669014b69..652307a1b22e 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c @@ -35,7 +35,7 @@ static int check_load(const char *file, enum bpf_prog_type type) } bpf_program__set_type(prog, type); - bpf_program__set_flags(prog, testing_prog_flags()); + bpf_program__set_flags(prog, bpf_program__flags(prog) | testing_prog_flags()); bpf_program__set_log_level(prog, 4 | extra_prog_load_log_flags); err = bpf_object__load(obj); diff --git a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c index 2e0ddef77ba5..ed3fd0a88dab 100644 --- a/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +++ b/tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c @@ -362,7 +362,7 @@ static void test_attach_api_fails(void) sl_skel->bss->user_ptr = sl_skel; err = bpf_program__set_flags(sl_skel->progs.handle_kprobe_multi_sleepable, - BPF_F_SLEEPABLE); + bpf_program__flags(sl_skel->progs.handle_kprobe_multi_sleepable) | BPF_F_SLEEPABLE); if (!ASSERT_OK(err, "sleep_skel_set_flags")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c index 5c31054ad4a4..047dfdc322a2 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c @@ -408,14 +408,14 @@ void test_xdp_metadata(void) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, rx_ifindex); - bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_XDP_DEV_BOUND_ONLY); /* Make sure we can load a dev-bound program that performs * XDP_REDIRECT into a devmap. */ new_prog = bpf_object__find_program_by_name(bpf_obj->obj, "redirect"); bpf_program__set_ifindex(new_prog, rx_ifindex); - bpf_program__set_flags(new_prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__set_flags(new_prog, bpf_program__flags(new_prog) | BPF_F_XDP_DEV_BOUND_ONLY); if (!ASSERT_OK(xdp_metadata__load(bpf_obj), "load skeleton")) goto out; diff --git a/tools/testing/selftests/bpf/xdp_hw_metadata.c b/tools/testing/selftests/bpf/xdp_hw_metadata.c index 6db3b5555a22..c5501b3fdf48 100644 --- a/tools/testing/selftests/bpf/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/xdp_hw_metadata.c @@ -845,7 +845,7 @@ int main(int argc, char *argv[]) prog = bpf_object__find_program_by_name(bpf_obj->obj, "rx"); bpf_program__set_ifindex(prog, ifindex); - bpf_program__set_flags(prog, BPF_F_XDP_DEV_BOUND_ONLY); + bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_XDP_DEV_BOUND_ONLY); printf("load bpf program...\n"); ret = xdp_hw_metadata__load(bpf_obj); From d6b633b585f348053dd27044b1b39972c748c1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 1 Sep 2026 10:46:56 +0200 Subject: [PATCH 77/90] libbpf: Move section-defined program flags to prog_flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The libbpf section definition modifiers for XDP frags support and sleepable programs stores the flags bits only in the private section definition cookie from object open to load time. This has the unfortunate consequence that API consumers cannot see (or manipulate) the flag between object open and program load. In particular, libxdp has special handling of frags-enabled programs to make them compatible with the dispatcher. This doesn't work on XDP programs that enable frags through the 'xdp.frags' section definition because the flag is not visible through bpf_program__flags()[0]. Fix this by changing how libbpf loads the program flags from section definitions: instead of using the private section definition cookie, add a setup function to the default section definitions that stores the flags for sleepable and XDP frags programs in the prog_flags field of struct bpf_program. Exposing the flags this way means that any use of bpf_program__set_flags() will override the flags unless the caller takes care of updating flags in a non-destructive way. This is unavoidable with the set-only API, and any user setting flags unconditionally is already broken in the sense that they will also override any other current and future flags. In addition, prog_flags survives program type changes through bpf_program__set_type(). It is the responsibility of the caller to ensure the flags are cleared if they are incompatible with the new program type. [0] https://github.com/xdp-project/xdp-tools/issues/587 Fixes: 082c4bfba4f7 ("libbpf: Add SEC name for xdp frags programs") Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260901-libbpf-frags-flags-v3-3-4eb6f14968b0@redhat.com --- tools/lib/bpf/libbpf.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index f09cbfd8e729..c036e8a91ed8 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -7879,6 +7879,19 @@ static int tracing_multi_mod_fd(struct bpf_program *prog, int *btf_obj_fd) return 0; } +static int libbpf_setup_prog_flags(struct bpf_program *prog, long cookie) +{ + enum sec_def_flags def = cookie; + + if (def & SEC_SLEEPABLE) + prog->prog_flags |= BPF_F_SLEEPABLE; + + if (def & SEC_XDP_FRAGS) + prog->prog_flags |= BPF_F_XDP_HAS_FRAGS; + + return 0; +} + /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ static int libbpf_prepare_prog_load(struct bpf_program *prog, struct bpf_prog_load_opts *opts, long cookie) @@ -7889,12 +7902,6 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) opts->expected_attach_type = 0; - if (def & SEC_SLEEPABLE) - opts->prog_flags |= BPF_F_SLEEPABLE; - - if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) - opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; - /* special check for usdt to use uprobe_multi link */ if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) { /* for BPF_TRACE_UPROBE_MULTI, user might want to query expected_attach_type @@ -10099,6 +10106,7 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts .prog_type = BPF_PROG_TYPE_##ptype, \ .expected_attach_type = atype, \ .cookie = (long)(flags), \ + .prog_setup_fn = libbpf_setup_prog_flags, \ .prog_prepare_load_fn = libbpf_prepare_prog_load, \ __VA_ARGS__ \ } From 99c71666dcb11b92bacedb7ddc5bf172aedc07c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toke=20H=C3=B8iland-J=C3=B8rgensen?= Date: Tue, 1 Sep 2026 10:46:57 +0200 Subject: [PATCH 78/90] selftests/bpf: Check for flag presence in bpf_program__flags() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a check that the BPF_F_XDP_HAS_FRAGS and BPF_F_SLEEPABLE flags show up in bpf_program__flags() when opening a BPF program with the flag definitions in their section definitions. Signed-off-by: Toke Høiland-Jørgensen Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260901-libbpf-frags-flags-v3-4-4eb6f14968b0@redhat.com --- tools/testing/selftests/bpf/prog_tests/kernel_flag.c | 3 +++ tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c | 3 +++ tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c index 97b00c7efe94..25eb59f460ab 100644 --- a/tools/testing/selftests/bpf/prog_tests/kernel_flag.c +++ b/tools/testing/selftests/bpf/prog_tests/kernel_flag.c @@ -16,6 +16,9 @@ void test_kernel_flag(void) if (!ASSERT_OK_PTR(lsm_skel, "lsm_skel")) return; + ASSERT_EQ(bpf_program__flags(lsm_skel->progs.bpf) & BPF_F_SLEEPABLE, + BPF_F_SLEEPABLE, "sleepable in program flags"); + lsm_skel->bss->monitored_tid = sys_gettid(); ret = test_kernel_flag__attach(lsm_skel); diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c index fce203640f8c..a894b1ab46f4 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c @@ -18,6 +18,9 @@ static void test_xdp_update_frags(void) return; prog = bpf_object__next_program(obj, NULL); + ASSERT_EQ(bpf_program__flags(prog) & BPF_F_XDP_HAS_FRAGS, + BPF_F_XDP_HAS_FRAGS, "frags in program flags"); + if (bpf_object__load(obj)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c index a8ab05216c38..dff6b3e7266e 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_devmap_attach.c @@ -146,6 +146,11 @@ static void test_xdp_with_devmap_frags_helpers(void) if (!ASSERT_OK_PTR(skel, "test_xdp_with_devmap_helpers__open_and_load")) return; + ASSERT_EQ(bpf_program__flags(skel->progs.xdp_dummy_dm_frags) & BPF_F_XDP_HAS_FRAGS, + BPF_F_XDP_HAS_FRAGS, "frags in program flags"); + ASSERT_EQ(bpf_program__flags(skel->progs.xdp_dummy_dm) & BPF_F_XDP_HAS_FRAGS, + 0, "frags not in program flags"); + dm_fd_frags = bpf_program__fd(skel->progs.xdp_dummy_dm_frags); map_fd = bpf_map__fd(skel->maps.dm_ports); err = bpf_prog_get_info_by_fd(dm_fd_frags, &info, &len); From 7bf591f26545f5776a4e42d08c06fd94469766dd Mon Sep 17 00:00:00 2001 From: Sudip Mukherjee Date: Mon, 31 Aug 2026 14:45:37 +0100 Subject: [PATCH 79/90] libbpf: Fix debian kernel version information parsing Currently get_debian_kernel_version() expects the version to be in the form of x.y.z but the Debian kernels uploaded to experimental are not always LTS kernels. Like the current Debian kernel in experimental is 7.2~rc7-1~exp1 and uname -v reported "Debian 7.2~rc7-1~exp1". Signed-off-by: Sudip Mukherjee Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20260831134537.1227914-1-sudipm.mukherjee@gmail.com --- tools/lib/bpf/libbpf_probes.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index e40819465ddc..5acf0f84a40a 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -62,7 +62,7 @@ static __u32 get_ubuntu_kernel_version(void) */ static __u32 get_debian_kernel_version(struct utsname *info) { - __u32 major, minor, patch; + __u32 major, minor, patch = 0; char *p; p = strstr(info->version, "Debian "); @@ -71,10 +71,13 @@ static __u32 get_debian_kernel_version(struct utsname *info) return 0; } - if (sscanf(p, "Debian %u.%u.%u", &major, &minor, &patch) != 3) - return 0; + if (sscanf(p, "Debian %u.%u.%u", &major, &minor, &patch) == 3) + return KERNEL_VERSION(major, minor, patch); - return KERNEL_VERSION(major, minor, patch); + if (sscanf(p, "Debian %u.%u", &major, &minor) == 2) + return KERNEL_VERSION(major, minor, patch); + + return 0; } __u32 get_kernel_version(void) From d02bfbc49c8ef978eac5f39a2de67281b8e00edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:30 +0200 Subject: [PATCH 80/90] bpf: mark instructions accessing program stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to prepare to emit KASAN checks in JITed programs, JIT compilers need to be aware about whether some load/store instructions are targeting the bpf program stack, as those should not be monitored (we already have guard pages for that, and it is difficult anyway to correctly monitor any kind of data passed on stack). To support this need, make the BPF verifier mark the instructions depending on whether they could access or not memory other than stack. As different states in the verifier could lead to different memory types for the same access, just marking an instruction as accessing stack only is not enough (it could be some other memory type in another verifier state), so the algorithm rather sets by default any load/store instruction as stack only, and if _any_ state leads to any memory access type other than PTR_TO_STACK, it overrides this setting. It also takes care about shifting back the instruction marking in adjust_insn_aux_data if the verifier patches instructions. However, if the verifier generates new BPF_ST/BPF_STX/BPF_LDX while patching some instructions, those new ones are systematically marked as non-stack-accessing: this may over-instrument a few memory accessing instructions, but it allows making sure that we will not miss accidentally any. Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-1-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 16 ++++++++++++++++ kernel/bpf/fixups.c | 33 ++++++++++++++++++++++++++++++--- kernel/bpf/verifier.c | 10 ++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ae9f606539f4..1a3c44ab06a1 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -706,6 +706,7 @@ struct bpf_insn_aux_data { */ u32 calls_callback:1; u32 indirect_target:1; /* if it is an indirect jump target */ + u32 non_stack_access:1; /* instruction can access non-stack memory */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -1671,6 +1672,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); } +static inline bool bpf_is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + if (insn->code == (BPF_ST | BPF_NOSPEC)) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + #define MAX_PACKET_OFF 0xffff #define CALLER_SAVED_REGS 6 diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 65b441e4a351..73fb3ffc18e3 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -213,7 +213,8 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_prog *new_prog, u32 off, u32 cnt) + struct bpf_prog *new_prog, u32 off, u32 cnt, + struct bpf_insn *original_insn) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -227,8 +228,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, */ data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; - if (cnt == 1) + if (cnt == 1) { + /* + * A non-memory accessing insn could have been replaced by a + * memory accessing insn, systematically mark it for non-stack + * access + */ + data[off].non_stack_access = bpf_is_mem_insn(insn + off); return; + } prog_len = new_prog->len; env->insn_aux_data_len = prog_len; @@ -239,8 +247,25 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; + if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + data[i].non_stack_access = + data[off + cnt - 1].non_stack_access; + data[off + cnt - 1].non_stack_access = false; + } else if (bpf_is_mem_insn(insn + i)) { + data[i].non_stack_access = true; + } } + /* + * Last slot instruction could be a newly generated + * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access + * if it is not the original instruction, otherwise keep the + * original marking + */ + if (bpf_is_mem_insn(insn + off + cnt - 1) && + memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) + data[off + cnt - 1].non_stack_access = true; + /* * The indirect_target flag of the original instruction was moved to the last of the * new instructions by the above memmove and memset, but the indirect jump target is @@ -306,6 +331,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + struct bpf_insn original_insn; if (len > 1) { new_data = vrealloc(env->insn_aux_data, @@ -318,6 +344,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data = new_data; } + memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); new_prog = bpf_patch_insn_single(env->prog, off, patch, len); if (IS_ERR(new_prog)) { if (PTR_ERR(new_prog) == -ERANGE) @@ -326,7 +353,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn); adjust_subprog_starts(env, off, len); adjust_insn_arrays(env, off, len); adjust_poke_descs(new_prog, off, len); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8f585ceb2cd5..5b51e7ee1a3f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3273,6 +3273,11 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } +static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].non_stack_access = true; +} + #define LR_FRAMENO_BITS 4 #define LR_SPI_BITS 6 #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) @@ -6417,6 +6422,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b int value_regno, bool strict_alignment_once, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); + enum bpf_reg_type ptr_type = reg->type; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6655,6 +6661,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b clear_scalar_id(®s[value_regno]); } } + + if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK) + mark_non_stack_access(env, insn_idx); + return err; } From 90769ad233cac2ade07f8d2f91214e584842158d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:31 +0200 Subject: [PATCH 81/90] bpf: add BPF_JIT_KASAN for KASAN instrumentation of JITed programs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new Kconfig option CONFIG_BPF_JIT_KASAN that automatically enables generic KASAN (Kernel Address SANitizer) memory access checks for JIT-compiled BPF programs as well, when both KASAN (and more specifically, generic KASAN with KASAN_VMALLOC) and JIT compiler are enabled. This new Kconfig is not a user selectable one: it is automatically enabled if KASAN is enabled on a compatible platform. When enabled, the JIT compiler will emit shadow memory checks before memory loads and stores to detect use-after-free or out-of-bounds accesses at runtime. The option is gated behind HAVE_EBPF_JIT_KASAN, as it needs proper arch-specific implementation. Acked-by: Andrey Konovalov Acked-by: Ihor Solodrai Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-2-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/Kconfig | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index eb3de35734f0..d7d25477ef48 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig @@ -17,6 +17,10 @@ config HAVE_CBPF_JIT config HAVE_EBPF_JIT bool +# KASAN support for JIT compiler +config HAVE_EBPF_JIT_KASAN + bool + # Used by archs to tell that they want the BPF JIT compiler enabled by # default for kernels that were compiled with BPF JIT support. config ARCH_WANT_DEFAULT_BPF_JIT @@ -101,4 +105,17 @@ config BPF_LSM If you are unsure how to answer this question, answer N. +config BPF_JIT_KASAN + bool + depends on HAVE_EBPF_JIT_KASAN + depends on KASAN_GENERIC + depends on KASAN_VMALLOC + depends on BPF_JIT + default y + help + Makes JIT compiler insert generic outline KASAN checks in BPF + programs when they are inserted in the kernel. This feature is + automatically enabled if the needed set of KASAN and BPF + configuration options is enabled. + endmenu # "BPF subsystem" From b3b45e9d72dfdc3534b04747978c891b0b30fc31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:32 +0200 Subject: [PATCH 82/90] bpf, x86: refactor BPF_ST management in do_jit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to prepare for KASAN checks insertion before every memory-related load or store, group all BPF_ST instructions that indeed access memory in a single helper to allow instrumenting those in one call, rather than having to instrument all cases individually. Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-3-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 73 +++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 48429fae0641..f907be3988b2 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1315,6 +1315,46 @@ static void emit_st_index(u8 **pprog, u32 size, u32 dst_reg, u32 index_reg, int *pprog = prog; } +/* ST: *(u8*)(dst_reg + off) = imm */ +static void emit_st(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, + s32 insn_off) +{ + s32 imm32 = insn->imm; + u8 *prog = *pprog; + + switch (BPF_SIZE(insn->code)) { + case BPF_B: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC6); + else + EMIT1(0xC6); + break; + case BPF_H: + if (is_ereg(dst_reg)) + EMIT3(0x66, 0x41, 0xC7); + else + EMIT2(0x66, 0xC7); + break; + case BPF_W: + if (is_ereg(dst_reg)) + EMIT2(0x41, 0xC7); + else + EMIT1(0xC7); + break; + case BPF_DW: + EMIT2(add_1mod(0x48, dst_reg), 0xC7); + break; + } + + if (is_imm8(insn_off)) + EMIT2(add_1reg(0x40, dst_reg), insn_off); + else + EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); + + EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + *pprog = prog; +} + static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) { emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm); @@ -2250,49 +2290,28 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_LFENCE(); break; - /* ST: *(u8*)(dst_reg + off) = imm */ case BPF_ST | BPF_MEM | BPF_B: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC6); - else - EMIT1(0xC6); - goto st; case BPF_ST | BPF_MEM | BPF_H: - if (is_ereg(dst_reg)) - EMIT3(0x66, 0x41, 0xC7); - else - EMIT2(0x66, 0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_W: - if (is_ereg(dst_reg)) - EMIT2(0x41, 0xC7); - else - EMIT1(0xC7); - goto st; case BPF_ST | BPF_MEM | BPF_DW: if (dst_reg == BPF_REG_PARAMS && insn->off == -8) { /* Arg 6: store immediate in r9 register */ - emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, (u32)imm32); + emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, + imm32); break; } - EMIT2(add_1mod(0x48, dst_reg), 0xC7); - -st: insn_off = insn->off; + insn_off = insn->off; if (dst_reg == BPF_REG_PARAMS) { /* * Args 7+: reverse BPF negative offsets to * x86 positive rsp offsets. * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ... */ - insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; + insn_off = outgoing_arg_base - outgoing_rsp - + insn_off - 16; dst_reg = BPF_REG_FP; } - if (is_imm8(insn_off)) - EMIT2(add_1reg(0x40, dst_reg), insn_off); - else - EMIT1_off32(add_1reg(0x80, dst_reg), insn_off); - - EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code))); + emit_st(&prog, insn, dst_reg, insn_off); break; /* STX: *(u8*)(dst_reg + off) = src_reg */ From 92b73b708b1764b3f3a02652c1e8dd3965b6d1ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:33 +0200 Subject: [PATCH 83/90] bpf, x86: emit KASAN checks in x86 JITed programs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Insert KASAN shadow memory checks before memory load and store operations in JIT-compiled BPF programs. This helps detect memory safety bugs such as use-after-free and out-of-bounds accesses at runtime. The main instructions being targeted are BPF_ST, BPF_STX and BPF_LDX, but not all of them are being instrumented: - if the load/store instruction is in fact accessing the program stack, emit_kasan_check silently skips the instrumentation, as we can already benefit from guard pages to monitor stack accesses. - if the load/store instruction is a BPF_PROBE_MEM or a BPF_PROBE_ATOMIC instruction, we do not instrument it, as the passed address can fault (hence the custom fault management with BPF_PROBE_XXX instructions), and so the corresponding kasan check could fault as well. To support those new instructions insertion, create the emit_kasan_check() helper that emits KASAN shadow memory checks before memory accesses in JIT-compiled BPF programs. The implementation relies on the existing __asan_{load,store}X functions from KASAN subsystem. The helper: - saves registers. This includes caller-saved registers, but also temporary registers, as those were possibly used by the affected program. - computes the accessed address and stores it in %rdi - calls the relevant function, depending on the instruction being a load or a store, and the size of the access. - restores registers The special care needed when inserting this instrumentation comes at the cost of a non negligeable increase in JITed code size. For example, a bare mov 0x0(%si),rbx # Load in rbx content at address stored in rsi becomes push %rax push %rcx push %rdx push %rsi push %rdi push %r8 push %r9 push %r10 push %r11 mov %rsi,%rdi call 0xffffffff81da0a60 <__asan_load8> pop %r11 pop %r10 pop %r9 pop %r8 pop %rdi pop %rsi pop %rdx pop %rcx pop %rax mov 0x0(%rsi),rbx Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-4-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 197 ++++++++++++++++++++++++++++++++---- 1 file changed, 179 insertions(+), 18 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index f907be3988b2..b8c2e935689e 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -21,6 +21,17 @@ #include #include +#if IS_ENABLED(CONFIG_BPF_JIT_KASAN) +void __asan_load1(void *p); +void __asan_store1(void *p); +void __asan_load2(void *p); +void __asan_store2(void *p); +void __asan_load4(void *p); +void __asan_store4(void *p); +void __asan_load8(void *p); +void __asan_store8(void *p); +#endif + static bool all_callee_regs_used[4] = {true, true, true, true}; static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) @@ -1110,6 +1121,93 @@ static void maybe_emit_1mod(u8 **pprog, u32 reg, bool is64) *pprog = prog; } +static int emit_kasan_check(struct bpf_verifier_env *env, u8 **pprog, + u32 addr_reg, s32 off, u32 bpf_size, u8 *ip, + bool is_write) +{ +#ifdef CONFIG_BPF_JIT_KASAN + u8 *prog = *pprog; + void *kasan_func; + + if (!env) + return 0; + + if (addr_reg == BPF_REG_FP || addr_reg == BPF_REG_PARAMS) + return 0; + + /* Derive KASAN check function from access type and size */ + switch (bpf_size) { + case BPF_B: + kasan_func = is_write ? __asan_store1 : __asan_load1; + break; + case BPF_H: + kasan_func = is_write ? __asan_store2 : __asan_load2; + break; + case BPF_W: + kasan_func = is_write ? __asan_store4 : __asan_load4; + break; + case BPF_DW: + kasan_func = is_write ? __asan_store8 : __asan_load8; + break; + default: + return -EINVAL; + } + + /* Save rax */ + EMIT1(0x50); + /* Save rcx */ + EMIT1(0x51); + /* Save rdx */ + EMIT1(0x52); + /* Save rsi */ + EMIT1(0x56); + /* Save rdi */ + EMIT1(0x57); + /* Save r8 */ + EMIT2(0x41, 0x50); + /* Save r9 */ + EMIT2(0x41, 0x51); + /* Save r10 */ + EMIT2(0x41, 0x52); + /* Save r11 */ + EMIT2(0x41, 0x53); + /* mov rdi, addr_reg */ + EMIT_mov(BPF_REG_1, addr_reg); + + /* add rdi, off (if offset is non-zero) */ + if (off) { + if (is_imm8(off)) { + /* add rdi, imm8 */ + EMIT4(0x48, 0x83, 0xC7, (u8)off); + } else { + /* add rdi, imm32 */ + EMIT3_off32(0x48, 0x81, 0xC7, off); + } + } + + /* Adjust ip to account for the instrumentation generated so far */ + ip += (prog - *pprog); + /* We emit a call, so update call depth counting */ + ip += x86_call_depth_emit_accounting(&prog, kasan_func, ip); + /* call kasan_func */ + if (emit_call(&prog, kasan_func, ip)) + return -ERANGE; + + EMIT2(0x41, 0x5B); + EMIT2(0x41, 0x5A); + EMIT2(0x41, 0x59); + EMIT2(0x41, 0x58); + EMIT1(0x5F); + EMIT1(0x5E); + EMIT1(0x5A); + EMIT1(0x59); + EMIT1(0x58); + + *pprog = prog; +#endif /* CONFIG_BPF_JIT_KASAN */ + return 0; +} + /* LDX: dst_reg = *(u8*)(src_reg + off) */ static void emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) { @@ -1463,17 +1561,35 @@ static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size, return 0; } -static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg, - u32 src_reg, s16 off, u8 bpf_size) +static int emit_atomic_ld_st(struct bpf_verifier_env *env, u8 **pprog, + struct bpf_insn *insn, u8 *ip, u32 dst_reg, + u32 src_reg, bool accesses_stack_only) { + u32 atomic_op = insn->imm; + int err; + switch (atomic_op) { case BPF_LOAD_ACQ: + if (!accesses_stack_only) { + err = emit_kasan_check(env, pprog, src_reg, insn->off, + BPF_SIZE(insn->code), ip, false); + if (err) + return err; + } /* dst_reg = smp_load_acquire(src_reg + off16) */ - emit_ldx(pprog, bpf_size, dst_reg, src_reg, off); + emit_ldx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; case BPF_STORE_REL: + if (!accesses_stack_only) { + err = emit_kasan_check(env, pprog, dst_reg, insn->off, + BPF_SIZE(insn->code), ip, true); + if (err) + return err; + } /* smp_store_release(dst_reg + off16, src_reg) */ - emit_stx(pprog, bpf_size, dst_reg, src_reg, off); + emit_stx(pprog, BPF_SIZE(insn->code), dst_reg, src_reg, + insn->off); break; default: pr_err("bpf_jit: unknown atomic load/store opcode %02x\n", @@ -1894,10 +2010,12 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * const s32 imm32 = insn->imm; u32 dst_reg = insn->dst_reg; u32 src_reg = insn->src_reg; + bool accesses_stack_only; u8 b2 = 0, b3 = 0; u8 *start_of_ldx; s64 jmp_offset; s32 insn_off; + int insn_idx; u8 jmp_cond; u8 *func; int nops; @@ -1914,6 +2032,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * EMIT_ENDBR(); ip = image + addrs[i - 1] + (prog - temp); + insn_idx = i - 1 + bpf_prog->aux->subprog_start; + accesses_stack_only = + env ? !env->insn_aux_data[insn_idx].non_stack_access : + false; switch (insn->code) { /* ALU */ @@ -2311,6 +2433,15 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * insn_off - 16; dst_reg = BPF_REG_FP; } + if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, dst_reg, + insn_off, + BPF_SIZE(insn->code), ip, + true); + if (err) + return err; + } + emit_st(&prog, insn, dst_reg, insn_off); break; @@ -2329,6 +2460,14 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16; dst_reg = BPF_REG_FP; } + if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, dst_reg, + insn_off, + BPF_SIZE(insn->code), ip, + true); + if (err) + return err; + } emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off); break; @@ -2510,6 +2649,13 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * /* populate jmp_offset for JAE above to jump to start_of_ldx */ start_of_ldx = prog; end_of_jmp[-1] = start_of_ldx - end_of_jmp; + } else if (!accesses_stack_only) { + err = emit_kasan_check(env, &prog, src_reg, + insn_off, + BPF_SIZE(insn->code), ip, + false); + if (err) + return err; } if (BPF_MODE(insn->code) == BPF_PROBE_MEMSX || BPF_MODE(insn->code) == BPF_MEMSX) @@ -2571,28 +2717,42 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * } fallthrough; case BPF_STX | BPF_ATOMIC | BPF_W: - case BPF_STX | BPF_ATOMIC | BPF_DW: - if (insn->imm == (BPF_AND | BPF_FETCH) || - insn->imm == (BPF_OR | BPF_FETCH) || - insn->imm == (BPF_XOR | BPF_FETCH)) { - bool is64 = BPF_SIZE(insn->code) == BPF_DW; - u32 real_src_reg = src_reg; - u32 real_dst_reg = dst_reg; - u8 *branch_target; - + case BPF_STX | BPF_ATOMIC | BPF_DW: { + bool is64 = BPF_SIZE(insn->code) == BPF_DW; + u32 real_src_reg = src_reg; + u32 real_dst_reg = dst_reg; + u8 *old_prog; + bool is_atomic_fetch = + (insn->imm == (BPF_AND | BPF_FETCH) || + insn->imm == (BPF_OR | BPF_FETCH) || + insn->imm == (BPF_XOR | BPF_FETCH)); + if (is_atomic_fetch) { /* * Can't be implemented with a single x86 insn. * Need to do a CMPXCHG loop. */ /* Will need RAX as a CMPXCHG operand so save R0 */ + old_prog = prog; emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); if (src_reg == BPF_REG_0) real_src_reg = BPF_REG_AX; if (dst_reg == BPF_REG_0) real_dst_reg = BPF_REG_AX; - - branch_target = prog; + ip += (prog - old_prog); + } + if (!bpf_atomic_is_load_store(insn)) { + if (!accesses_stack_only) { + err = emit_kasan_check( + env, &prog, real_dst_reg, + insn->off, BPF_SIZE(insn->code), + ip, true); + if (err) + return err; + } + } + if (is_atomic_fetch) { + u8 *branch_target = prog; /* Load old value */ emit_ldx(&prog, BPF_SIZE(insn->code), BPF_REG_0, real_dst_reg, insn->off); @@ -2624,15 +2784,16 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * } if (bpf_atomic_is_load_store(insn)) - err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg, - insn->off, BPF_SIZE(insn->code)); + err = emit_atomic_ld_st(env, &prog, insn, ip, + dst_reg, src_reg, + accesses_stack_only); else err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg, insn->off, BPF_SIZE(insn->code)); if (err) return err; break; - + } case BPF_STX | BPF_PROBE_ATOMIC | BPF_B: case BPF_STX | BPF_PROBE_ATOMIC | BPF_H: if (!bpf_atomic_is_load_store(insn)) { From cfc821c029fd66237b9a1f66362d93d4a5cab48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:34 +0200 Subject: [PATCH 84/90] bpf, x86: enable KASAN for JITed programs on x86 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mark x86 as supporting KASAN checks in JITed programs so that the corresponding JIT compiler inserts checks on the translated instructions. Acked-by: Ihor Solodrai Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-5-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- arch/x86/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 48ccc3e6059d..745890d91e99 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -234,6 +234,7 @@ config X86 select HAVE_SAMPLE_FTRACE_DIRECT if X86_64 select HAVE_SAMPLE_FTRACE_DIRECT_MULTI if X86_64 select HAVE_EBPF_JIT + select HAVE_EBPF_JIT_KASAN if X86_64 select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_EISA if X86_32 select HAVE_EXIT_THREAD From 040ef1f8b02e57b78081581d490ceec04ef5f7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:35 +0200 Subject: [PATCH 85/90] selftests/bpf: make cmdline_contains stricter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cmdline_contains is used by BPF selftests to check the presence of specific kernel commandline parameters, but it currently suffers from two issues: - the read commandline isn't NULL terminated right after the read data but only at the end of the buffer, leaving uninitialized bytes that are then possibly tokenized - the comparison of found tokens is done based on the size of found token. This could lead to too-short-but-matching tokens to wrongly match the search pattern. Enforce stricter checks in cmdline_contains to avoid accidental matches. Fixes: 399f6185a1c0 ("selftests/bpf: Fix selftests broken by mitigations=off") Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-6-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/unpriv_helpers.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/unpriv_helpers.c b/tools/testing/selftests/bpf/unpriv_helpers.c index f997d7ec8fd0..9dadcacaef0c 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.c +++ b/tools/testing/selftests/bpf/unpriv_helpers.c @@ -72,8 +72,8 @@ static int config_contains(const char *pat) static bool cmdline_contains(const char *pat) { + int fd, cnt, ret = false; char cmdline[4096], *c; - int fd, ret = false; fd = open("/proc/cmdline", O_RDONLY); if (fd < 0) { @@ -81,14 +81,15 @@ static bool cmdline_contains(const char *pat) return false; } - if (read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + cnt = read(fd, cmdline, sizeof(cmdline) - 1); + if (cnt < 0) { perror("read /proc/cmdline"); goto out; } - cmdline[sizeof(cmdline) - 1] = '\0'; + cmdline[cnt] = '\0'; for (c = strtok(cmdline, " \n"); c; c = strtok(NULL, " \n")) { - if (strncmp(c, pat, strlen(c))) + if (strcmp(c, pat)) continue; ret = true; break; From 9123a4d4c4f3321efbb40569b81985d9654abe25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:36 +0200 Subject: [PATCH 86/90] selftests/bpf: add helpers for KASAN in JIT testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add two simple helpers to allow checking whether KASAN for eBPF tests should be executed: - one helper to check if BPF_JIT_KASAN is enabled in kernel configuration - one helper to check if the kernel is running with kasan_multi_shot (otherwise only the first test will be able to trigger a report) Acked-by: Ihor Solodrai Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-7-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/unpriv_helpers.c | 10 ++++++++++ tools/testing/selftests/bpf/unpriv_helpers.h | 2 ++ 2 files changed, 12 insertions(+) diff --git a/tools/testing/selftests/bpf/unpriv_helpers.c b/tools/testing/selftests/bpf/unpriv_helpers.c index 9dadcacaef0c..2c8c5edb8751 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.c +++ b/tools/testing/selftests/bpf/unpriv_helpers.c @@ -143,3 +143,13 @@ bool get_unpriv_disabled(void) } return mitigations_off; } + +bool get_kasan_jit_enabled(void) +{ + return config_contains("CONFIG_BPF_JIT_KASAN=y") == 1; +} + +bool get_kasan_multi_shot_enabled(void) +{ + return cmdline_contains("kasan_multi_shot"); +} diff --git a/tools/testing/selftests/bpf/unpriv_helpers.h b/tools/testing/selftests/bpf/unpriv_helpers.h index 151f67329665..a7ceb51577cd 100644 --- a/tools/testing/selftests/bpf/unpriv_helpers.h +++ b/tools/testing/selftests/bpf/unpriv_helpers.h @@ -5,3 +5,5 @@ #define UNPRIV_SYSCTL "kernel/unprivileged_bpf_disabled" bool get_unpriv_disabled(void); +bool get_kasan_jit_enabled(void); +bool get_kasan_multi_shot_enabled(void); From 2cbbb035977a79b78952404d3e2c3c6dfe318259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20Lothor=C3=A9=20=28eBPF=20Foundation=29?= Date: Thu, 3 Sep 2026 15:51:37 +0200 Subject: [PATCH 87/90] selftests/bpf: add tests to validate KASAN on JIT programs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a basic KASAN test runner that loads and test-run programs that can trigger memory management bugs. The test captures kernel logs and ensure that the expected KASAN splat is emitted by searching for the corresponding first lines in the report, hence validated that the needed instrumentation has been inserted by the JIT compiler before the relevant memory accesses. To allow each test to trigger the expected report, the kernel must run with the kasan_multi_shot configuration. The runner covers different cases and settings: in the nominal case, it validates kasan reports on basic instructions (on all supported accesses sizes) but also when report _should not_ be emitted (eg: for accesses on program stack). The runner also comes with a few specialized tests that are then not executed for all sizes/locations: - specific atomic ops - test for instructions involving different verifier states, with some states flagging memory as stack, and other states as non-stack memory - tests that validate the stack marking shifting when a patch is emitted by the verifier (zext/rnd_hi32, constant blindind). Most of those tests are able to trigger kasan reports by altering the shadow memory (triggering faulty accesses is otherwise complex, because of the verifier). A few tests trigger actual faulty accesses (eg out-of-bound accesses) A few of those tests depends on cpuv4 (load_acquire/store_release, st, st_blinded), and so are executed only with test_progs-cpuv4 # ./test_progs -a kasan #175/1 kasan/st_1_not_on_stack:SKIP #175/2 kasan/st_1_on_stack:SKIP #175/3 kasan/st_2_not_on_stack:SKIP #175/4 kasan/st_2_on_stack:SKIP #175/5 kasan/st_4_not_on_stack:SKIP #175/6 kasan/st_4_on_stack:SKIP #175/7 kasan/st_8_not_on_stack:SKIP #175/8 kasan/st_8_on_stack:SKIP #175/9 kasan/stx_1_not_on_stack:OK #175/10 kasan/stx_1_on_stack:OK #175/11 kasan/stx_2_not_on_stack:OK #175/12 kasan/stx_2_on_stack:OK #175/13 kasan/stx_4_not_on_stack:OK #175/14 kasan/stx_4_on_stack:OK #175/15 kasan/stx_8_not_on_stack:OK #175/16 kasan/stx_8_on_stack:OK #175/17 kasan/ldx_1_not_on_stack:OK #175/18 kasan/ldx_1_on_stack:OK #175/19 kasan/ldx_2_not_on_stack:OK #175/20 kasan/ldx_2_on_stack:OK #175/21 kasan/ldx_4_not_on_stack:OK #175/22 kasan/ldx_4_on_stack:OK #175/23 kasan/ldx_8_not_on_stack:OK #175/24 kasan/ldx_8_on_stack:OK #175/25 kasan/simple_atomic_4_not_on_stack:OK #175/26 kasan/simple_atomic_4_on_stack:OK #175/27 kasan/simple_atomic_8_not_on_stack:OK #175/28 kasan/simple_atomic_8_on_stack:OK #175/29 kasan/simple_atomic_fetch_not_on_stack:OK #175/30 kasan/simple_atomic_fetch_on_stack:OK #175/31 kasan/load_acquire_1_not_on_stack:SKIP #175/32 kasan/load_acquire_1_on_stack:SKIP #175/33 kasan/load_acquire_2_not_on_stack:SKIP #175/34 kasan/load_acquire_2_on_stack:SKIP #175/35 kasan/load_acquire_4_not_on_stack:SKIP #175/36 kasan/load_acquire_4_on_stack:SKIP #175/37 kasan/load_acquire_8_not_on_stack:SKIP #175/38 kasan/load_acquire_8_on_stack:SKIP #175/39 kasan/store_release_1_not_on_stack:SKIP #175/40 kasan/store_release_1_on_stack:SKIP #175/41 kasan/store_release_2_not_on_stack:SKIP #175/42 kasan/store_release_2_on_stack:SKIP #175/43 kasan/store_release_4_not_on_stack:SKIP #175/44 kasan/store_release_4_on_stack:SKIP #175/45 kasan/store_release_8_not_on_stack:SKIP #175/46 kasan/store_release_8_on_stack:SKIP #175/47 kasan/ldx_patched_not_on_stack:OK #175/48 kasan/ldx_patched_on_stack:OK #175/49 kasan/verifier_paths_stack_and_non_stack:OK #175/50 kasan/ldx_oob_1_not_on_stack:OK #175/51 kasan/ldx_oob_2_not_on_stack:OK #175/52 kasan/ldx_oob_4_not_on_stack:OK #175/53 kasan/ldx_oob_8_not_on_stack:OK #175/54 kasan/ldx_self_alias_on_stack:OK #175/55 kasan/st_blinded:SKIP #175 kasan:OK (SKIP: 25/55) Summary: 1/30 PASSED, 25 SKIPPED, 0/0 FAILED Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-8-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- .../testing/selftests/bpf/prog_tests/kasan.c | 479 +++++++++++++++++ tools/testing/selftests/bpf/progs/kasan.c | 502 ++++++++++++++++++ .../selftests/bpf/progs/kasan_harden.c | 52 ++ .../selftests/bpf/test_kmods/bpf_testmod.c | 55 ++ 4 files changed, 1088 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/kasan.c create mode 100644 tools/testing/selftests/bpf/progs/kasan.c create mode 100644 tools/testing/selftests/bpf/progs/kasan_harden.c diff --git a/tools/testing/selftests/bpf/prog_tests/kasan.c b/tools/testing/selftests/bpf/prog_tests/kasan.c new file mode 100644 index 000000000000..7cd4d1208c3b --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/kasan.c @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +/* + * Tests validating that KASAN reports are properly instrumented and + * generated on a wide variety of instructions. The running kernel needs + * kasan_multi_shot to run multiple kasan-generating subtests at once + */ +#include +#include +#include +#include +#include +#include +#include +#include "sysctl_helpers.h" +#include "kasan.skel.h" +#include "kasan_harden.skel.h" + +#define SUBTEST_NAME_MAX_LEN 128 +#define PROG_NAME_MAX_LEN 128 + +#define MAX_LOG_SIZE (8 * 1024) +#define READ_CHUNK_SIZE 256 + +#define KASAN_PATTERN_SLAB_UAF "BUG: KASAN: slab-use-after-free " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_SLAB_OOB "BUG: KASAN: slab-out-of-bounds " \ + "in bpf_prog_%02x%02x%02x%02x%02x%02x%02x%02x_%s" +#define KASAN_PATTERN_REPORT "%s of size %d at addr" + +static char klog_buffer[MAX_LOG_SIZE]; +static char record[MAX_LOG_SIZE]; + +struct test_spec { + char *prog_type; + bool is_write; + bool only_32_or_64; + bool needs_load_acq_store_rel; + bool needs_st; + bool skip_multi_size_testing; + bool skip_on_stack_testing; + int run_size; + bool expect_no_report; + bool rnd_hi32; + bool is_oob; +}; + +struct kasan_write_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct test_ctx { + __u8 prog_tag[BPF_TAG_SIZE]; + struct bpf_object *obj; + int *access_size; + bool skip_load_acq_store_rel; + bool skip_st_tests; + struct bpf_program *prog; + char prog_name[SUBTEST_NAME_MAX_LEN]; + int klog_fd; +}; + +static int open_kernel_logs(void) +{ + int fd; + + fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); + + return fd; +} + +static void skip_kernel_logs(int fd) +{ + lseek(fd, 0, SEEK_END); +} + +static int read_kernel_logs(int fd, char *buf, size_t max_len) +{ + size_t total = 0; + ssize_t n; + + buf[0] = '\0'; + while (1) { + char *msg, *eol; + size_t len; + + n = read(fd, record, sizeof(record) - 1); + if (n == 0) + break; + + if (n < 0) { + if (errno == EAGAIN) + break; + return n; + } + record[n] = '\0'; + + /* + * Each kmsg record starts with some metadata, separated + * from the actual content by a semi-colon + */ + msg = strchr(record, ';'); + if (!msg) + continue; + msg++; + eol = strchr(msg, '\n'); + if (eol) + *eol = '\0'; + + len = strlen(msg); + if (total + len + 2 > max_len) + break; + memcpy(buf + total, msg, len); + total += len; + buf[total++] = '\n'; + buf[total] = '\0'; + } + + return total; +} + +static int check_kasan_report_in_kernel_logs(char *buf, struct test_ctx *ctx, + bool is_write, int size, + bool is_oob) +{ + char access_log[READ_CHUNK_SIZE]; + const char *pattern; + char *kasan_report_start; + int nsize; + + pattern = is_oob ? KASAN_PATTERN_SLAB_OOB : KASAN_PATTERN_SLAB_UAF; + nsize = snprintf(access_log, READ_CHUNK_SIZE, pattern, + ctx->prog_tag[0], ctx->prog_tag[1], ctx->prog_tag[2], + ctx->prog_tag[3], ctx->prog_tag[4], ctx->prog_tag[5], + ctx->prog_tag[6], ctx->prog_tag[7], ctx->prog_name); + if (!ASSERT_GE(nsize, 0, "format kasan access header line")) + return nsize; + /* + * Searched kasan report is valid if + * - it contains the expected kasan pattern + * - the description of the faulty access is found somewhere + * after the header (not necessarily on the very next line, + * because other kernel messages may interleave) + * - faulty access properties match the tested type and size + */ + kasan_report_start = strstr(buf, access_log); + + if (!kasan_report_start) + return 1; + + nsize = snprintf(access_log, READ_CHUNK_SIZE, KASAN_PATTERN_REPORT, + is_write ? "Write" : "Read", size); + if (!ASSERT_GE(nsize, 0, "format kasan access report line")) + return nsize; + + if (!strstr(kasan_report_start, access_log)) + return 1; + + return 0; +} + +static void exec_subtest(struct test_ctx *ctx, struct test_spec *test, + int access_size, bool on_stack) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + struct bpf_prog_info info; + uint8_t buf[ETH_HLEN] = {0}; + int ret, prog_fd; + __u32 info_len; + + ctx->prog = bpf_object__find_program_by_name(ctx->obj, + ctx->prog_name); + if (!ASSERT_OK_PTR(ctx->prog, "find test prog")) + return; + + info_len = sizeof(info); + memset(&info, 0, info_len); + prog_fd = bpf_program__fd(ctx->prog); + if (!ASSERT_OK_FD(prog_fd, "get prog fd")) + return; + ret = bpf_prog_get_info_by_fd(prog_fd, &info, &info_len); + if (!ASSERT_OK(ret, "fetch loaded program info")) + return; + memcpy(ctx->prog_tag, info.tag, BPF_TAG_SIZE); + + skip_kernel_logs(ctx->klog_fd); + + topts.sz = sizeof(struct bpf_test_run_opts); + topts.data_size_in = ETH_HLEN; + topts.data_in = buf; + if (ctx->access_size) + *ctx->access_size = access_size; + ret = bpf_prog_test_run_opts(bpf_program__fd(ctx->prog), + &topts); + if (!ASSERT_OK(ret, "run prog")) + return; + + ret = read_kernel_logs(ctx->klog_fd, klog_buffer, MAX_LOG_SIZE); + if (!ASSERT_GE(ret, 0, "read kernel logs")) + return; + + ret = check_kasan_report_in_kernel_logs(klog_buffer, ctx, + test->is_write, access_size, + test->is_oob); + if (on_stack || test->expect_no_report) + ASSERT_NEQ(ret, 0, "no report should be generated"); + else + ASSERT_OK(ret, "report should be generated"); +} + +static void run_subtest_with_size_and_location(struct test_ctx *ctx, + struct test_spec *test, + int access_size, + bool on_stack) +{ + char subtest_name[SUBTEST_NAME_MAX_LEN]; + + if (test->skip_multi_size_testing) { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s%s", + test->prog_type, + test->skip_on_stack_testing ? "" : + on_stack ? "_on_stack" : + "_not_on_stack"); + } else { + snprintf(subtest_name, SUBTEST_NAME_MAX_LEN, "%s_%d_%s", + test->prog_type, access_size, + on_stack ? "on_stack" : "not_on_stack"); + } + + snprintf(ctx->prog_name, PROG_NAME_MAX_LEN, "%s%s", test->prog_type, + test->skip_on_stack_testing ? "" : + on_stack ? "_on_stack" : + "_not_on_stack"); + + if (!test__start_subtest(subtest_name)) + return; + + if (test->needs_load_acq_store_rel && ctx->skip_load_acq_store_rel) { + test__skip(); + return; + } + + if (test->needs_st && ctx->skip_st_tests) { + test__skip(); + return; + } + + exec_subtest(ctx, test, access_size, on_stack); +} + +static void run_subtest_with_size(struct test_ctx *ctx, struct test_spec *test, + int size) +{ + run_subtest_with_size_and_location(ctx, test, size, false); + if (!test->skip_on_stack_testing) + run_subtest_with_size_and_location(ctx, test, size, true); +} + +static void run_subtest(struct test_ctx *ctx, struct test_spec *test) +{ + if (test->skip_multi_size_testing) { + run_subtest_with_size(ctx, test, test->run_size); + return; + } + + if (!test->only_32_or_64) { + run_subtest_with_size(ctx, test, 1); + run_subtest_with_size(ctx, test, 2); + } + run_subtest_with_size(ctx, test, 4); + run_subtest_with_size(ctx, test, 8); +} + +static void run_blinding_subtest(void) +{ + struct test_spec blinding_spec = { + .prog_type = "st_blinded", + .is_write = true, + }; + char bpf_jit_harden_orig[2]; + struct kasan_harden *skel; + struct test_ctx *ctx; + + if (!test__start_subtest("st_blinded")) + return; + + ctx = calloc(1, sizeof(*ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc blinding ctx")) + return; + ctx->klog_fd = -1; + + if (sysctl_set_or_fail("/proc/sys/net/core/bpf_jit_harden", + bpf_jit_harden_orig, "2")) + goto free_ctx; + + skel = kasan_harden__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open and load blinded prog")) + goto restore; + + if (skel->data->skip_st_tests) { + test__skip(); + goto destroy; + } + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + ctx->obj = skel->obj; + strncpy(ctx->prog_name, "st_blinded", PROG_NAME_MAX_LEN); + + exec_subtest(ctx, &blinding_spec, 1, false); + +destroy: + close(ctx->klog_fd); + kasan_harden__destroy(skel); +restore: + sysctl_set_or_fail("/proc/sys/net/core/bpf_jit_harden", NULL, + bpf_jit_harden_orig); +free_ctx: + free(ctx); +} + +static struct test_spec tests[] = { + { + .prog_type = "st", + .is_write = true, + .needs_st = true + }, + { + .prog_type = "stx", + .is_write = true + }, + { + .prog_type = "ldx", + .is_write = false + }, + { + .prog_type = "simple_atomic", + .is_write = true, + .only_32_or_64 = true + }, + { + .prog_type = "simple_atomic_fetch", + .is_write = true, + .skip_multi_size_testing = true, + .run_size = 8, + }, + { + .prog_type = "load_acquire", + .is_write = false, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "store_release", + .is_write = true, + .needs_load_acq_store_rel = true + }, + { + .prog_type = "ldx_patched", + .is_write = false, + .skip_multi_size_testing = true, + .run_size = 4, + .rnd_hi32 = true + }, + { + .prog_type = "verifier_paths_stack_and_non_stack", + .is_write = true, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 1 + }, + { + .prog_type = "ldx_oob", + .is_write = false, + .skip_on_stack_testing = true, + .is_oob = true + }, + { + .prog_type = "ldx_self_alias_on_stack", + .is_write = false, + .skip_multi_size_testing = true, + .skip_on_stack_testing = true, + .run_size = 8, + .expect_no_report = true + } +}; + +void serial_test_kasan(void) +{ + struct kasan_write_val val; + struct test_spec *test; + struct test_ctx *ctx; + struct kasan *skel; + __u32 key = 0; + int i, ret; + + ctx = calloc(1, sizeof(struct test_ctx)); + if (!ASSERT_OK_PTR(ctx, "alloc test ctx")) + return; + + if (!is_jit_enabled() || !get_kasan_jit_enabled() || + !get_kasan_multi_shot_enabled()) { + test__skip(); + goto end; + } + + skel = kasan__open(); + if (!ASSERT_OK_PTR(skel, "open prog")) + goto end; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + char prog_name[SUBTEST_NAME_MAX_LEN]; + struct bpf_program *prog; + + if (!tests[i].rnd_hi32) + continue; + + snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", + tests[i].prog_type, "on_stack"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 on_stack prog")) + goto destroy; + bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32); + snprintf(prog_name, SUBTEST_NAME_MAX_LEN, "%s_%s", + tests[i].prog_type, "not_on_stack"); + prog = bpf_object__find_program_by_name(skel->obj, prog_name); + if (!ASSERT_OK_PTR(prog, "find rnd_hi32 not_on_stack prog")) + goto destroy; + bpf_program__set_flags(prog, BPF_F_TEST_RND_HI32); + } + + if (!ASSERT_OK(kasan__load(skel), "load prog")) + goto destroy; + + ctx->obj = skel->obj; + ctx->access_size = &skel->bss->access_size; + ctx->skip_load_acq_store_rel = skel->data->skip_load_acq_store_rel_tests; + ctx->skip_st_tests = skel->data->skip_st_tests; + + ctx->klog_fd = open_kernel_logs(); + if (!ASSERT_OK_FD(ctx->klog_fd, "open kernel logs")) + goto destroy; + + /* Fill map with recognizable values */ + ret = bpf_map__lookup_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "get map")) + goto close; + val.data_1 = 0xAA; + val.data_2 = 0xBBBB; + val.data_4 = 0xCCCCCCCC; + val.data_8 = 0xDDDDDDDDDDDDDDDD; + ret = bpf_map__update_elem(skel->maps.test_map, &key, sizeof(key), + &val, sizeof(val), 0); + if (!ASSERT_OK(ret, "set map")) + goto close; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + test = &tests[i]; + run_subtest(ctx, test); + } + + /* + * Blinding subtest is handled differently as it needs the + * corresponding program to be loaded with bpf_jit_harden raised + */ + run_blinding_subtest(); + +close: + close(ctx->klog_fd); +destroy: + kasan__destroy(skel); +end: + free(ctx); +} diff --git a/tools/testing/selftests/bpf/progs/kasan.c b/tools/testing/selftests/bpf/progs/kasan.c new file mode 100644 index 000000000000..fe8e0dd228a0 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan.c @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include +#include +#include +#include +#include "bpf_misc.h" + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct bpf_testmod_oob { + __u8 data; + union { + __u8 redzone_1; + __u16 redzone_2; + __u32 redzone_4; + __u64 redzone_8; + }; +}; + +extern struct bpf_testmod_oob *bpf_testmod_oob_alloc(void) __ksym; +extern void bpf_testmod_oob_free(struct bpf_testmod_oob *oob) __ksym; + +int access_size; + +struct kasan_test_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_test_val); +} test_map SEC(".maps"); + +/* + * ST instructions are only emitted if the BPF cpu supports it (eg cpuv4), + * they are otherwise turned into MOV + STX, so compile and exercise ST + * only if supported. + */ +#ifdef __BPF_FEATURE_ST +SEC("tcx/ingress") +int st_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val.data_1 = 0xAA; + break; + case 2: + val.data_2 = 0xAA; + break; + case 4: + val.data_4 = 0xAA; + break; + case 8: + val.data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int st_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val->data_1 = 0xAA; + break; + case 2: + val->data_2 = 0xAA; + break; + case 4: + val->data_4 = 0xAA; + break; + case 8: + val->data_8 = 0xAA; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +bool skip_st_tests SEC(".data") = 0; +#else +bool skip_st_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int stx_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + /* + * Unlike the st() programs above, the stored value comes from a + * runtime source (skb->len), so it cannot be constant-folded and + * clang always emits a genuine BPF_STX (register store) regardless + * of the target cpu version. + */ + switch (access_size) { + case 1: + val.data_1 = (__u8)skb->len; + break; + case 2: + val.data_2 = (__u16)skb->len; + break; + case 4: + val.data_4 = (__u32)skb->len; + break; + case 8: + val.data_8 = (__u64)skb->len; + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int stx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + val->data_1 = (__u8)skb->len; + break; + case 2: + val->data_2 = (__u16)skb->len; + break; + case 4: + val->data_4 = (__u32)skb->len; + break; + case 8: + val->data_8 = (__u64)skb->len; + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __sink(val.data_1); + break; + case 2: + __sink(val.data_2); + break; + case 4: + __sink(val.data_4); + break; + case 8: + __sink(val.data_8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __sink(val->data_1); + break; + case 2: + __sink(val->data_2); + break; + case 4: + __sink(val->data_4); + break; + case 8: + __sink(val->data_8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + __sink(val->data_4); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + + return 0; +} + +SEC("tcx/ingress") +int ldx_patched_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + __sink(val.data_4); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val.data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val.data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 4: + __sync_fetch_and_add(&val->data_4, 4); + break; + case 8: + __sync_fetch_and_add(&val->data_8, 8); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_fetch_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + __sync_fetch_and_or(&val.data_8, 8); + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int simple_atomic_fetch_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + __sync_fetch_and_or(&val->data_8, 8); + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +#ifdef __BPF_FEATURE_LOAD_ACQ_STORE_REL +bool skip_load_acq_store_rel_tests SEC(".data") = 0; + +SEC("tcx/ingress") +int load_acquire_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val.data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val.data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val.data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val.data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int load_acquire_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_load_n(&val->data_1, __ATOMIC_ACQUIRE); + break; + case 2: + __atomic_load_n(&val->data_2, __ATOMIC_ACQUIRE); + break; + case 4: + __atomic_load_n(&val->data_4, __ATOMIC_ACQUIRE); + break; + case 8: + __atomic_load_n(&val->data_8, __ATOMIC_ACQUIRE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + + bpf_kfunc_kasan_poison(&val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val.data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val.data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val.data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val.data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(&val, sizeof(struct kasan_test_val)); + return 0; +} + +SEC("tcx/ingress") +int store_release_not_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + switch (access_size) { + case 1: + __atomic_store_n(&val->data_1, 0xAA, __ATOMIC_RELEASE); + break; + case 2: + __atomic_store_n(&val->data_2, 0xBBBB, __ATOMIC_RELEASE); + break; + case 4: + __atomic_store_n(&val->data_4, 0xCCCCCCCC, __ATOMIC_RELEASE); + break; + case 8: + __atomic_store_n(&val->data_8, 0xDDDDDDDDDDDDDDDD, + __ATOMIC_RELEASE); + break; + } + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + return 0; +} +#else +bool skip_load_acq_store_rel_tests SEC(".data") = 1; +#endif + +SEC("tcx/ingress") +int verifier_paths_stack_and_non_stack(struct __sk_buff *skb) +{ + struct kasan_test_val stack_val = {}; + struct kasan_test_val *val; + void *ptr; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + if (access_size) + ptr = val; + else + ptr = &stack_val; + + bpf_kfunc_kasan_poison(val, sizeof(*val)); + *(__u8 *)ptr = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(*val)); + return 0; +} + +SEC("tcx/ingress") +int ldx_oob(struct __sk_buff *skb) +{ + struct bpf_testmod_oob *val; + struct kasan_test_val volatile tmp; + + val = bpf_testmod_oob_alloc(); + if (!val) + return 0; + + switch (access_size) { + case 1: + tmp.data_1 = (__u8)val->redzone_1; + break; + case 2: + tmp.data_2 = (__u16)val->redzone_2; + break; + case 4: + tmp.data_4 = (__u32)val->redzone_4; + break; + case 8: + tmp.data_8 = (__u64)val->redzone_8; + break; + } + bpf_testmod_oob_free(val); + return tmp.data_1; +} + +SEC("tcx/ingress") +int ldx_self_alias_on_stack(struct __sk_buff *skb) +{ + struct kasan_test_val val; + __u64 addr; + + bpf_kfunc_kasan_poison(&val, sizeof(val)); + /* + * Check that a stack access with dst_reg == src_reg is correctly + * flagged as stack-only access + */ + addr = (__u64)&val; + asm volatile( + "r1 = %0\n" + "r1 = *(u64 *)(r1 + 0)\n" + : + : "r"(addr) + : "r1", "memory"); + bpf_kfunc_kasan_unpoison(&val, sizeof(val)); + + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/kasan_harden.c b/tools/testing/selftests/bpf/progs/kasan_harden.c new file mode 100644 index 000000000000..8c9eb203419c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/kasan_harden.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause + +#include +#include +#include +#include + +extern void bpf_kfunc_kasan_poison(void *mem, __u32 mem__sz) __ksym; +extern void bpf_kfunc_kasan_unpoison(void *mem, __u32 mem__sz) __ksym; + +struct kasan_test_val { + __u8 data_1; + __u16 data_2; + __u32 data_4; + __u64 data_8; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct kasan_test_val); +} test_map SEC(".maps"); + +/* + * ST instructions are only emitted if the BPF cpu supports it (eg cpuv4), + * they are otherwise turned into MOV + STX, so compile and exercise ST + * only if supported. + */ +#ifdef __BPF_FEATURE_ST +SEC("tcx/ingress") +int st_blinded(struct __sk_buff *skb) +{ + struct kasan_test_val *val; + __u32 key = 0; + + val = bpf_map_lookup_elem(&test_map, &key); + if (!val) + return 0; + + bpf_kfunc_kasan_poison(val, sizeof(struct kasan_test_val)); + val->data_1 = 0xAA; + bpf_kfunc_kasan_unpoison(val, sizeof(struct kasan_test_val)); + + return 0; +} +bool skip_st_tests SEC(".data") = 0; +#else +bool skip_st_tests SEC(".data") = 1; +#endif + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 2380b6cbdead..f798bbbb4d13 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -75,6 +75,16 @@ union bpf_testmod_union_arg_2 { struct bpf_testmod_struct_arg_2 arg; }; +struct bpf_testmod_oob { + __u8 data; + union { + __u8 redzone_1; + __u16 redzone_2; + __u32 redzone_4; + __u64 redzone_8; + }; +}; + __bpf_hook_start(); noinline int @@ -336,6 +346,47 @@ __bpf_kfunc void bpf_kfunc_put_default_trusted_ptr_test(struct prog_test_member */ } +#ifdef CONFIG_BPF_JIT_KASAN + +extern void kasan_poison(const void *addr, size_t size, u8 value, bool init); + +#define KASAN_SLAB_FREE 0xFB + +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, KASAN_SLAB_FREE, false); +} + +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) +{ + kasan_poison(mem, mem__sz, 0x00, false); +} +#else +__bpf_kfunc void bpf_kfunc_kasan_poison(void *mem, u32 mem__sz) { } +__bpf_kfunc void bpf_kfunc_kasan_unpoison(void *mem, u32 mem__sz) { } +#endif + +__bpf_kfunc struct bpf_testmod_oob *bpf_testmod_oob_alloc(void) +{ + struct bpf_testmod_oob *p; + + /* + * Only allocate size of data (and so, voluntarily use kmalloc + * instead of kmalloc_obj), not the rest of the structure, so + * that programs under test trying to access the rest of the + * structure trigger OoB accesses + */ + p = kmalloc(sizeof(p->data), GFP_ATOMIC); + if (!p) + return NULL; + return p; +} + +__bpf_kfunc void bpf_testmod_oob_free(struct bpf_testmod_oob *oob) +{ + kfree(oob); +} + __bpf_kfunc struct bpf_testmod_ctx * bpf_testmod_ctx_create(int *err) { @@ -869,6 +920,10 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack) BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot) BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test); BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test); +BTF_ID_FLAGS(func, bpf_kfunc_kasan_poison) +BTF_ID_FLAGS(func, bpf_kfunc_kasan_unpoison) +BTF_ID_FLAGS(func, bpf_testmod_oob_alloc, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_testmod_oob_free, KF_RELEASE) BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids) BTF_ID_LIST(bpf_testmod_dtor_ids) From f80a09c486e7c2a0e4ca43dbff7570df668484c5 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Sat, 5 Sep 2026 08:58:55 +0000 Subject: [PATCH 88/90] riscv, bpf: Remap BPF_REG_0 and RV_REG_TCC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To prepare for supporting bpf stack arguments and unifying the calling convention between bpf2bpf calls and kfunc calls, remap BPF_REG_0 to t6 and RV_REG_TCC to t5. In the riscv abi, a0-a7 are designated for function argument passing. Freeing a5 and a6 allows bpf2bpf calls to pass extra arguments (6th to 8th) via a5-a7 consistently with kfuncs. Signed-off-by: Pu Lehui Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260905085857.14143-2-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit_comp64.c | 10 +++++----- arch/riscv/net/bpf_timed_may_goto.S | 8 ++++---- .../testing/selftests/bpf/progs/verifier_jit_inline.c | 2 +- tools/testing/selftests/bpf/progs/verifier_ldsx.c | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index e7378be171a9..fc1c8c5480b6 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -23,11 +23,11 @@ /* fentry and TCC init insns will be skipped on tailcall */ #define RV_TAILCALL_OFFSET ((RV_FENTRY_NINSNS + 1) * 4) -#define RV_REG_TCC RV_REG_A6 +#define RV_REG_TCC RV_REG_T5 #define RV_REG_ARENA RV_REG_S7 /* For storing arena_vm_start */ static const int regmap[] = { - [BPF_REG_0] = RV_REG_A5, + [BPF_REG_0] = RV_REG_T6, [BPF_REG_1] = RV_REG_A0, [BPF_REG_2] = RV_REG_A1, [BPF_REG_3] = RV_REG_A2, @@ -47,13 +47,13 @@ static const int pt_regmap[] = { [RV_REG_A2] = offsetof(struct pt_regs, a2), [RV_REG_A3] = offsetof(struct pt_regs, a3), [RV_REG_A4] = offsetof(struct pt_regs, a4), - [RV_REG_A5] = offsetof(struct pt_regs, a5), [RV_REG_S1] = offsetof(struct pt_regs, s1), [RV_REG_S2] = offsetof(struct pt_regs, s2), [RV_REG_S3] = offsetof(struct pt_regs, s3), [RV_REG_S4] = offsetof(struct pt_regs, s4), [RV_REG_S5] = offsetof(struct pt_regs, s5), [RV_REG_T0] = offsetof(struct pt_regs, t0), + [RV_REG_T6] = offsetof(struct pt_regs, t6), }; enum { @@ -239,7 +239,7 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) emit_addi(RV_REG_SP, RV_REG_SP, stack_adjust, ctx); /* Set return value. */ if (!is_tail_call) - emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); + emit_addiw(RV_REG_A0, regmap[BPF_REG_0], 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, is_tail_call ? RV_TAILCALL_OFFSET : 0, ctx); } @@ -1075,7 +1075,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); if (save_ret) - stack_size += 16; /* Save both A5 (BPF R0) and A0 */ + stack_size += 16; /* Save both RV_REG_T6 (BPF R0) and RV_REG_A0 */ retval_off = stack_size; stack_size += nr_arg_slots * 8; diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index 02c637d87420..fcf850f244c9 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -10,7 +10,7 @@ * - output: updated count in BPF_REG_AX (t0) * * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (a5, a0-a4) are saved + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. */ @@ -20,8 +20,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S s0, 6*SZREG(sp) addi s0, sp, 8*SZREG - /* Save BPF registers R0-R5 (a5, a0-a4) */ - REG_S a5, 5*SZREG(sp) + /* Save BPF registers R0-R5 (t6, a0-a4) */ + REG_S t6, 5*SZREG(sp) REG_S a0, 4*SZREG(sp) REG_S a1, 3*SZREG(sp) REG_S a2, 2*SZREG(sp) @@ -38,7 +38,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_L a2, 2*SZREG(sp) REG_L a1, 3*SZREG(sp) REG_L a0, 4*SZREG(sp) - REG_L a5, 5*SZREG(sp) + REG_L t6, 5*SZREG(sp) REG_L s0, 6*SZREG(sp) REG_L ra, 7*SZREG(sp) diff --git a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c index 02e562f56f9d..5b92da2ab75c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jit_inline.c +++ b/tools/testing/selftests/bpf/progs/verifier_jit_inline.c @@ -11,7 +11,7 @@ __jited(" addq %gs:{{.*}}, %rax") __arch_arm64 __jited(" mrs x8, SP_EL0") __arch_riscv64 -__jited(" mv a5, tp") +__jited(" mv t6, tp") __arch_loongarch __jited(" move $a5, $tp") int inline_bpf_get_current_task(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_ldsx.c b/tools/testing/selftests/bpf/progs/verifier_ldsx.c index 7e25e1b8cd0b..49e8cd4bb4f9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ldsx.c +++ b/tools/testing/selftests/bpf/progs/verifier_ldsx.c @@ -289,11 +289,11 @@ __jited("ldrsh x22, [x11, #0x18]") __jited("add x11, x0, x28") __jited("ldrsb x22, [x11, #0x20]") __arch_riscv64 -__jited("add t2, a5, s7") +__jited("add t2, t6, s7") __jited("lw s3, 0x10(t2)") -__jited("add t2, a5, s7") +__jited("add t2, t6, s7") __jited("lh s3, 0x18(t2)") -__jited("add t2, a5, s7") +__jited("add t2, t6, s7") __jited("lb s3, 0x20(t2)") __jited("add t2, a0, s7") __jited("lw s4, 0x10(t2)") From ba754185f90291a0a164ec9adf7ba19737f18d3f Mon Sep 17 00:00:00 2001 From: Feng Jiang Date: Sat, 5 Sep 2026 08:58:56 +0000 Subject: [PATCH 89/90] riscv, bpf: Add BPF stack arguments support for RV64 JIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement bpf_jit_supports_stack_args() on RV64 JIT to allow bpf subprogs and kfuncs to pass and receive more than 5 arguments. In the riscv abi, the first 8 arguments are passed in registers a0 to a7, and arguments 9+ reside on the stack. To align bpf stack arguments with this calling convention and unify bpf2bpf calls with kfuncs, map the first 3 bpf stack arguments (6th to 8th) directly to a5 to a7, and store or load the remaining arguments (9th+) at SP or FP. Reserve outgoing stack space in the prologue accordingly when stack_arg_cnt exceeds 3. In addition, update kfunc argument sign extension to handle all 8 register arguments as well as any arguments passed on the stack. Co-developed-by: Pu Lehui Signed-off-by: Feng Jiang Signed-off-by: Pu Lehui Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260905085857.14143-3-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- arch/riscv/net/bpf_jit.h | 4 ++ arch/riscv/net/bpf_jit_comp64.c | 78 +++++++++++++++++++++++++++++---- arch/riscv/net/bpf_jit_core.c | 7 +++ 3 files changed, 80 insertions(+), 9 deletions(-) diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 419b9d795f2a..5c4f53060c91 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -12,6 +12,9 @@ #include #include +#define RV_MAX_REG_ARGS 8 +#define RV_EXTRA_STK_ARGS (RV_MAX_REG_ARGS - MAX_BPF_FUNC_REG_ARGS) + /* verify runtime detection extension status */ #define rv_ext_enabled(ext) \ (IS_ENABLED(CONFIG_RISCV_ISA_##ext) && riscv_has_extension_likely(RISCV_ISA_EXT_##ext)) @@ -82,6 +85,7 @@ struct rv_jit_context { unsigned long flags; int stack_size; int tcc_offset; + int stack_arg_sz; u64 arena_vm_start; u64 user_vm_start; }; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index fc1c8c5480b6..7f84447a31ff 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -15,7 +15,6 @@ #include #include "bpf_jit.h" -#define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) /* imm that allows emit_imm to emit max count insns */ @@ -498,6 +497,18 @@ static void emit_ldx(u8 rd, s16 off, u8 rs, u8 size, bool sign_ext, ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_ldx(u8 rd, s16 off, struct rv_jit_context *ctx) +{ + int idx = off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_mv(rd, RV_REG_A5 + idx, ctx); + return; + } + + emit_ldx_insn(rd, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_FP, BPF_DW, false, ctx); +} + static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx) { emit_imm(RV_REG_T1, imm, ctx); @@ -515,6 +526,19 @@ static void emit_st(u8 rd, s16 off, s32 imm, u8 size, struct rv_jit_context *ctx ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_st(s16 off, s32 imm, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_imm(RV_REG_A5 + idx, imm, ctx); + return; + } + + emit_imm(RV_REG_T1, imm, ctx); + emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, RV_REG_T1, BPF_DW, ctx); +} + static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) { if (is_12b_int(off)) { @@ -531,6 +555,18 @@ static void emit_stx(u8 rd, s16 off, u8 rs, u8 size, struct rv_jit_context *ctx) ctx->ex_jmp_off = ctx->ninsns; } +static void emit_stack_arg_stx(s16 off, u8 rs, struct rv_jit_context *ctx) +{ + int idx = -off / 8 - 1; + + if (idx < RV_EXTRA_STK_ARGS) { + emit_mv(RV_REG_A5 + idx, rs, ctx); + return; + } + + emit_stx_insn(RV_REG_SP, (idx - RV_EXTRA_STK_ARGS) * 8, rs, BPF_DW, ctx); +} + static int emit_atomic_ld_st(u8 rd, u8 rs, const struct bpf_insn *insn, struct rv_jit_context *ctx) { @@ -1824,11 +1860,21 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, return -EINVAL; for (idx = 0; idx < fm->nr_args; idx++) { - u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; + u8 arg_sz = fm->arg_size[idx]; - if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) - return -EINVAL; + if (arg_sz == 8 || (arg_sz != 4 && !sign)) + continue; + + if (idx < RV_MAX_REG_ARGS) { + if (sign_extend(RV_REG_A0 + idx, RV_REG_A0 + idx, arg_sz, sign, ctx)) + return -EINVAL; + } else { + emit_ld(RV_REG_T1, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_SP, ctx); + if (sign_extend(RV_REG_T1, RV_REG_T1, arg_sz, sign, ctx)) + return -EINVAL; + emit_sd(RV_REG_SP, (idx - RV_MAX_REG_ARGS) * 8, RV_REG_T1, ctx); + } } } @@ -1927,7 +1973,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rs = RV_REG_T2; } - emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); + if (is_stack_arg_ldx(insn)) + emit_stack_arg_ldx(rd, off, ctx); + else + emit_ldx(rd, off, rs, BPF_SIZE(code), sign_ext, ctx); ret = add_exception_handler(insn, rd, ctx); if (ret) @@ -1957,7 +2006,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T3; } - emit_st(rd, off, imm, BPF_SIZE(code), ctx); + if (is_stack_arg_st(insn)) + emit_stack_arg_st(off, imm, ctx); + else + emit_st(rd, off, imm, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -1979,7 +2031,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, rd = RV_REG_T2; } - emit_stx(rd, off, rs, BPF_SIZE(code), ctx); + if (is_stack_arg_stx(insn)) + emit_stack_arg_stx(off, rs, ctx); + else + emit_stx(rd, off, rs, BPF_SIZE(code), ctx); ret = add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); if (ret) @@ -2029,9 +2084,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { - int i, stack_adjust = 0, store_offset, bpf_stack_adjust; + int i, stack_adjust = 0, store_offset, bpf_stack_adjust = ctx->stack_arg_sz; - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); + bpf_stack_adjust += round_up(ctx->prog->aux->stack_depth, STACK_ALIGN); if (bpf_stack_adjust) mark_fp(ctx); @@ -2190,3 +2245,8 @@ bool bpf_jit_supports_timed_may_goto(void) { return true; } + +bool bpf_jit_supports_stack_args(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index cbfcd287ea16..2fb0b4e198b9 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -48,6 +48,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; + u16 stack_arg_cnt; if (!prog->jit_requested) return prog; @@ -71,6 +72,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr ctx->arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); + + stack_arg_cnt = bpf_out_stack_arg_cnt(env, prog); + /* First 3 stack args in regs, rest on stack */ + ctx->stack_arg_sz = stack_arg_cnt > RV_EXTRA_STK_ARGS ? + round_up((stack_arg_cnt - RV_EXTRA_STK_ARGS) * 8, STACK_ALIGN) : 0; + ctx->prog = prog; ctx->offset = kvzalloc_objs(int, prog->len); if (!ctx->offset) From 7687f1cbbaf8f17cddce4dd5c23a610c5c9a17f3 Mon Sep 17 00:00:00 2001 From: Feng Jiang Date: Sat, 5 Sep 2026 08:58:57 +0000 Subject: [PATCH 90/90] selftests/bpf: Enable stack argument tests for RV64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable the stack argument selftests for RV64. The riscv guard is restricted to __riscv_xlen == 64 since stack arguments are only supported by the RV64 JIT. Keep the __BPF_FEATURE_STACK_ARGUMENT gate, defined by clang with BPF stack argument support [1]. [1] https://github.com/llvm/llvm-project/pull/189060 Signed-off-by: Feng Jiang Acked-by: Björn Töpel Link: https://lore.kernel.org/bpf/20260905085857.14143-4-pulehui@huaweicloud.com Signed-off-by: Kumar Kartikeya Dwivedi --- tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c | 3 ++- .../selftests/bpf/progs/btf__verifier_stack_arg_order.c | 3 ++- tools/testing/selftests/bpf/progs/stack_arg.c | 3 ++- tools/testing/selftests/bpf/progs/stack_arg_kfunc.c | 3 ++- tools/testing/selftests/bpf/progs/stack_arg_precision.c | 3 ++- tools/testing/selftests/bpf/progs/verifier_stack_arg.c | 3 ++- tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c | 3 ++- 7 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c index 8d38aafe66a2..ab12397a9b52 100644 --- a/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c @@ -4,7 +4,8 @@ #include #include "../test_kmods/bpf_testmod_kfunc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) long subprog_call_mem_kfunc(long a, long b, long c, long d, long e, long size) diff --git a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c index 99bc115f8380..a980a3a399d3 100644 --- a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c @@ -3,7 +3,8 @@ #include #include -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) int subprog_bad_order_6args(int a, int b, int c, int d, int e, int f) diff --git a/tools/testing/selftests/bpf/progs/stack_arg.c b/tools/testing/selftests/bpf/progs/stack_arg.c index 944e3bb603e7..3e9ed37c57bb 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg.c +++ b/tools/testing/selftests/bpf/progs/stack_arg.c @@ -21,7 +21,8 @@ struct { int timer_result; -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) const volatile bool has_stack_arg = true; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c index 345f2da2e361..b6b1a8aa7f44 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_kfunc.c @@ -6,7 +6,8 @@ #include "bpf_kfuncs.h" #include "../test_kmods/bpf_testmod_kfunc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) const volatile bool has_stack_arg = true; diff --git a/tools/testing/selftests/bpf/progs/stack_arg_precision.c b/tools/testing/selftests/bpf/progs/stack_arg_precision.c index ce0301a41fa9..7baa9c1c8e31 100644 --- a/tools/testing/selftests/bpf/progs/stack_arg_precision.c +++ b/tools/testing/selftests/bpf/progs/stack_arg_precision.c @@ -6,7 +6,8 @@ #include "../test_kmods/bpf_testmod_kfunc.h" #include "bpf_misc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) /* Force kfunc extern BTF generation for inline asm call below. diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index 51d22faf4559..f6a4a6aa0e61 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -12,7 +12,8 @@ struct { __type(value, long long); } map_hash_8b SEC(".maps"); -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) __noinline __used diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index 8e4325273b3d..57f22691744a 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -5,7 +5,8 @@ #include #include "bpf_misc.h" -#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && \ +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ + (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) __noinline __used __naked