From 05bee0638bacf4e9298bbf0bfef749b6a8ff4478 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 19 Oct 2026 20:11:06 +0000 Subject: [PATCH 1/3] bpf: Revalidate faulted arena pages before installing user mappings A user fault on an arena pins the page that the kernel arena maps, drops the arena spinlock and returns the page to the core fault code, which then locks the folio and installs the user PTE. A concurrent free can clear the kernel mapping in that window. The page reference prevents a use-after-free, but the fault can still install a user PTE for a page the arena has already released. User space and BPF programs then see different memory at the same arena offset. Close the window from the fault side. Lock the page, recheck that the kernel mapping still points to it, and return it with VM_FAULT_LOCKED so that it stays locked until the core fault code has installed the user PTE. If a free removed the page, drop it and return VM_FAULT_NOPAGE so that the fault starts over. Return VM_FAULT_HWPOISON for a poisoned page. This change alone does not close the race, because the free path does not yet wait for the page lock. The following change makes frees wait for in-flight faults before zapping user mappings. Fixes: 317460317a02 ("bpf: Introduce bpf_arena.") Suggested-by: Alexei Starovoitov Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019201108.4552-2-emil@etsalapatis.com --- kernel/bpf/arena.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 0ff707707da0..5e0e0837980b 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -584,8 +584,22 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); if (new_page) free_pages_nolock(new_page, 0); - vmf->page = page; - return 0; + + /* + * A free clears the kernel pte and then takes the page lock before it + * zaps user vma-s. Either it waits for this fault and zaps its pte, or + * the page is gone from here. + */ + lock_page(page); + if (vmalloc_to_page((void *)kaddr) == page && !PageHWPoison(page)) { + vmf->page = page; + return VM_FAULT_LOCKED; + } + /* The page was freed meanwhile. Let the fault start over. */ + fault_ret = PageHWPoison(page) ? VM_FAULT_HWPOISON : VM_FAULT_NOPAGE; + unlock_page(page); + put_page(page); + return fault_ret; out_err_locked_memcg: bpf_map_memcg_exit(old_memcg, new_memcg); From 7244a2e58d2167292972f9380abb3a4d17a343e3 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 19 Oct 2026 20:11:07 +0000 Subject: [PATCH 2/3] bpf: Fix arena race between page free and alloc leading to incoherency Existing arena kfunc code has an underlying race condition that can lead to writes being lost from the BPF program's point of view: a) A memory range gets freed by operation (1), and its range is added back to the arena range tree. b) A concurrent allocation (2) reallocates the range, and does writes to it. Writes from that CPU may follow the stale TLB entries into the pages that are about to be freed. c) (1) invalidates the TLB. The old pages, and any writes done to them, are now inaccessible. zap_pages() similarly removes the mappings for userspace threads. This can be triggered by particularly demanding BPF arena data structures that constantly allocate and deallocate memory, like hash table allocations. Solve this ABA problem by keeping the range allocated until TLB invalidation and unmapping are complete. Under the arena spinlock, claim any free pages in the range and clear its kernel PTEs. Then drop the spinlock, lock and unlock each cleared page to wait for faults that are still installing a user PTE, flush the kernel TLB and zap the user page tables. Finally, retake the spinlock to release the range. Frees serialize on zap_mutex, so an overlapping free cannot release the range early. Claiming the range only allocates a range tree node when the whole range is free, and then there is nothing to free. User faults on a range that is allocated without a page now raise SIGBUS instead of populating it. That covers a range reserved with bpf_arena_reserve_pages() and a range whose kernel PTEs a free has already cleared. Touching a page while a BPF program frees it is a user space bug, and in a BPF_F_SEGV_ON_FAULT arena it already raises SIGSEGV. Sleepable frees run synchronously and are deferred only when the arena spinlock cannot be taken. The deferred free worker retries such a free in place instead of requeueing itself, so the flush_work() in arena_map_free() waits for all of its work. Fixes: b8467290edab ("bpf: arena: make arena kfuncs any context safe") Reported-by: Mykola Lysenko Suggested-by: Alexei Starovoitov Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019201108.4552-3-emil@etsalapatis.com --- kernel/bpf/arena.c | 159 +++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 86 deletions(-) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 5e0e0837980b..868589f07330 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -543,6 +543,12 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) goto out_err_locked; } + if (is_range_tree_set(&arena->rt, vmf->pgoff, 1)) { + /* Allocated without a page: reserved or being freed */ + fault_ret = VM_FAULT_SIGBUS; + goto out_err_locked; + } + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); if (!new_page) { @@ -868,7 +874,7 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) * otherwise call B's mark would make call A skip a vml that A has * not yet zapped for A's uaddr range. */ - mutex_lock(&arena->zap_mutex); + lockdep_assert_held(&arena->zap_mutex); mutex_lock(&arena->lock); my_gen = ++arena->zap_gen; for (;;) { @@ -902,49 +908,41 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt) mutex_lock(&arena->lock); } mutex_unlock(&arena->lock); - mutex_unlock(&arena->zap_mutex); } -static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +/* + * The range stays allocated in the range tree until the pages are unmapped + * from vmalloc area and from all user space vma-s, so that nobody can + * populate it while stale TLB entries or user ptes are still around. + */ +static int arena_free_range(struct bpf_arena *arena, long uaddr, long page_cnt) { - struct mem_cgroup *new_memcg, *old_memcg; - u64 full_uaddr, uaddr_end; - long kaddr, pgoff; - struct page *page; - struct llist_head free_pages; - struct llist_node *pos, *t; - struct arena_free_span *s; + u64 full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + long kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; + long pgoff = compute_pgoff(arena, uaddr); struct clear_range_data cdata; + struct llist_node *pos, *t; + LLIST_HEAD(free_pages); unsigned long flags; - int ret = 0; + struct page *page; + int ret; - /* only aligned lower 32-bit are relevant */ - uaddr = (u32)uaddr; - uaddr &= PAGE_MASK; - kaddr = bpf_arena_get_kern_vm_start(arena) + uaddr; - full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; - if (full_uaddr < arena->user_vm_start) - return; - uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); - if (full_uaddr >= uaddr_end) - return; - - page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; - pgoff = compute_pgoff(arena, uaddr); - bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - - if (!sleepable) - goto defer; + /* An overlapping free must not release the range before this one is done */ + guard(mutex)(&arena->zap_mutex); ret = raw_res_spin_lock_irqsave(&arena->spinlock, flags); - - /* Can't proceed without holding the spinlock so defer the free */ if (ret) - goto defer; + return ret; - range_tree_set(&arena->rt, pgoff, page_cnt); + /* + * Take the free pages of the range too. It allocates, and can fail, + * only when the whole range is free. Then there is nothing to do. + */ + if (range_tree_clear(&arena->rt, pgoff, page_cnt)) { + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; + } - init_llist_head(&free_pages); cdata.arena = arena; cdata.free_pages = &free_pages; /* clear ptes and collect struct pages */ @@ -954,6 +952,12 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, /* drop the lock to do the tlb flush and zap pages */ raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + /* wait for arena_vm_fault() that found the page before its pte was cleared */ + llist_for_each_entry(page, free_pages.first, pcp_llist) { + lock_page(page); + unlock_page(page); + } + /* ensure no stale TLB entries */ flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); @@ -972,8 +976,39 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, zap_pages(arena, full_uaddr, 1); __free_page(page); } - bpf_map_memcg_exit(old_memcg, new_memcg); + /* The callers hold no other rqspinlock, so this one can only time out */ + while (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) + cond_resched(); + range_tree_set(&arena->rt, pgoff, page_cnt); + raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); + return 0; +} + +static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) +{ + struct mem_cgroup *new_memcg, *old_memcg; + u64 full_uaddr, uaddr_end; + struct arena_free_span *s; + + /* only aligned lower 32-bit are relevant */ + uaddr = (u32)uaddr; + uaddr &= PAGE_MASK; + full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; + uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); + if (full_uaddr >= uaddr_end) + return; + + page_cnt = (uaddr_end - full_uaddr) >> PAGE_SHIFT; + bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); + + /* Can't proceed without holding the spinlock so defer the free */ + if (!sleepable || arena_free_range(arena, uaddr, page_cnt)) + goto defer; + + bpf_map_memcg_exit(old_memcg, new_memcg); return; defer: @@ -1035,66 +1070,18 @@ static void arena_free_worker(struct work_struct *work) { struct bpf_arena *arena = container_of(work, struct bpf_arena, free_work); struct mem_cgroup *new_memcg, *old_memcg; - struct llist_node *list, *pos, *t; + struct llist_node *pos, *t; struct arena_free_span *s; - u64 arena_vm_start, user_vm_start; - struct llist_head free_pages; - struct clear_range_data cdata; - struct page *page; - unsigned long full_uaddr; - long kaddr, page_cnt, pgoff; - unsigned long flags; - - if (raw_res_spin_lock_irqsave(&arena->spinlock, flags)) { - schedule_work(work); - return; - } bpf_map_memcg_enter(&arena->map, &old_memcg, &new_memcg); - init_llist_head(&free_pages); - cdata.arena = arena; - cdata.free_pages = &free_pages; - arena_vm_start = bpf_arena_get_kern_vm_start(arena); - user_vm_start = bpf_arena_get_user_vm_start(arena); - - list = llist_del_all(&arena->free_spans); - llist_for_each(pos, list) { + llist_for_each_safe(pos, t, llist_del_all(&arena->free_spans)) { s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - kaddr = arena_vm_start + s->uaddr; - pgoff = compute_pgoff(arena, s->uaddr); - - /* clear ptes and collect pages in free_pages llist */ - apply_to_existing_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT, - apply_range_clear_cb, &cdata); - - range_tree_set(&arena->rt, pgoff, page_cnt); - } - raw_res_spin_unlock_irqrestore(&arena->spinlock, flags); - - /* Iterate the list again without holding spinlock to do the tlb flush and zap_pages */ - llist_for_each_safe(pos, t, list) { - s = llist_entry(pos, struct arena_free_span, node); - page_cnt = s->page_cnt; - full_uaddr = clear_lo32(user_vm_start) + s->uaddr; - kaddr = arena_vm_start + s->uaddr; - - /* ensure no stale TLB entries */ - flush_tlb_kernel_range(kaddr, kaddr + (page_cnt * PAGE_SIZE)); - - /* remove pages from user vmas */ - zap_pages(arena, full_uaddr, page_cnt); - + while (arena_free_range(arena, s->uaddr, s->page_cnt)) + cond_resched(); kfree_nolock(s); } - /* free all pages collected by apply_to_existing_page_range() in the first loop */ - llist_for_each_safe(pos, t, __llist_del_all(&free_pages)) { - page = llist_entry(pos, struct page, pcp_llist); - __free_page(page); - } - bpf_map_memcg_exit(old_memcg, new_memcg); } From 6c372031078cfa4c1ee1a789d29108af794ec1ab Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Mon, 19 Oct 2026 20:11:08 +0000 Subject: [PATCH 3/3] selftests/bpf: Add arena allocation race tests Add selftests for races between arena frees, reallocation and user faults: - realloc_during_free, realloc_during_deferred_free: a synchronous or deferred free keeps its range allocated until it completes. - fault_free_realloc: user and BPF views of a page stay coherent while it is repeatedly faulted, freed and reallocated. Pause frees in zap_vma_range(), after the range's kernel mappings are gone. Unlike flush_tlb_kernel_range(), which is inline on arm64, it is generic MM code that fentry can attach to. Map the page into user space first, so that the free has to zap it. Print why a subtest is skipped when a tracing target is missing. The reader in fault_free_realloc skips reads that raise SIGBUS, which a fault on a page that is being freed does. Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20261019201108.4552-4-emil@etsalapatis.com --- .../selftests/bpf/prog_tests/arena_race.c | 320 ++++++++++++++++++ .../testing/selftests/bpf/progs/arena_race.c | 166 +++++++++ 2 files changed, 486 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/arena_race.c create mode 100644 tools/testing/selftests/bpf/progs/arena_race.c diff --git a/tools/testing/selftests/bpf/prog_tests/arena_race.c b/tools/testing/selftests/bpf/prog_tests/arena_race.c new file mode 100644 index 000000000000..f9d0e9c3a006 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_race.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include "arena_race.skel.h" + +struct free_thread_ctx { + struct arena_race *skel; + int err; + __u32 retval; +}; + +struct fault_thread_ctx { + __u64 *addr; + int stop; +}; + +static int run_prog(struct bpf_program *prog, const char *name) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts); + return ASSERT_OK(err, name) && ASSERT_OK(opts.retval, name) ? 0 : -1; +} + +/* Trigger the sleepable free page path. */ +static void *run_free_thread(void *arg) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct free_thread_ctx *ctx = arg; + + ctx->skel->bss->target_tid = sys_gettid(); + ctx->err = bpf_prog_test_run_opts( + bpf_program__fd(ctx->skel->progs.free_page), &opts); + ctx->retval = opts.retval; + return NULL; +} + +static __thread sigjmp_buf reader_env; +static __thread bool reader_armed; + +/* Only the reader expects SIGBUS; anywhere else it keeps its default action. */ +static void reader_sigbus(int sig) +{ + if (!reader_armed) { + signal(sig, SIG_DFL); + raise(sig); + return; + } + siglongjmp(reader_env, 1); +} + +/* Continuously fault in the address. */ +static void *fault_reader_thread(void *arg) +{ + struct fault_thread_ctx *ctx = arg; + __u64 *addr; + + /* Touching a page while BPF frees it raises SIGBUS; skip that read. */ + sigsetjmp(reader_env, 1); + reader_armed = true; + while (!READ_ONCE(ctx->stop)) { + addr = __atomic_load_n(&ctx->addr, __ATOMIC_ACQUIRE); + (void)READ_ONCE(*addr); + } + reader_armed = false; + return NULL; +} + +static int wait_for(int *p) +{ + __u64 deadline = get_time_ns() + 5ULL * 1000 * 1000 * 1000; + + while (!READ_ONCE(*p)) { + if (get_time_ns() > deadline) + return -ETIMEDOUT; + } + return 0; +} + +/* Skip the subtest, saying why, if @func cannot be traced with fentry. */ +static bool require_fentry_target(const char *test, const char *func) +{ + if (libbpf_find_vmlinux_btf_id(func, BPF_TRACE_FENTRY) > 0) + return true; + printf("%s:SKIP: %s is not an fentry target\n", test, func); + test__skip(); + return false; +} + +static struct arena_race *setup_arena(__u64 **addr, bool trace_zap) +{ + struct arena_race *skel; + size_t arena_sz; + char *base; + int err; + + skel = arena_race__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return NULL; + err = bpf_program__set_autoload(skel->progs.trace_zap, trace_zap); + if (!ASSERT_OK(err, "trace_zap_autoload")) + goto err_out; + + err = arena_race__load(skel); + if (!ASSERT_OK(err, "load")) + goto err_out; + err = arena_race__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto err_out; + + if (run_prog(skel->progs.alloc_old, "alloc_old")) + goto err_out; + if (skel->bss->skip) { + printf("%s:SKIP: compiler doesn't support arena_cast\n", __func__); + test__skip(); + goto err_out; + } + + base = bpf_map__initial_value(skel->maps.arena, &arena_sz); + if (!ASSERT_OK_PTR(base, "arena_base")) + goto err_out; + *addr = (__u64 *)(base + getpagesize()); + if (!ASSERT_EQ((unsigned long)skel->bss->ptr, (unsigned long)*addr, + "arena_ptr")) + goto err_out; + return skel; + +err_out: + arena_race__destroy(skel); + return NULL; +} + +static void test_realloc_during_free(bool deferred) +{ + struct free_thread_ctx ctx = {}; + struct arena_race *skel; + pthread_t thread; + __u64 *addr; + bool thread_created = false, zap_seen = false, completed = false; + int err; + + if (!require_fentry_target(__func__, "zap_vma_range")) + return; + + skel = setup_arena(&addr, true); + if (!skel) + return; + + /* Map the page into user space, so that the free has to zap it. */ + if (!ASSERT_EQ(READ_ONCE(*addr), skel->rodata->old_marker, "old_marker")) + goto out; + + /* Pause the free after its TLB flush, before it releases the range. */ + skel->bss->pause_on_zap = 1; + + if (deferred) { + /* + * Test the nonsleepable free path that gets + * deferred to a worker in the kernel. We do + * so by triggering the arena operation from + * a nonsleepable tracepoint context. + */ + skel->bss->trigger_pid_tgid = + ((__u64)getpid() << 32) | (__u32)sys_gettid(); + skel->bss->trigger_syscall = SYS_getpgid; + skel->bss->deferred_free = 1; + if (!ASSERT_GE(syscall(SYS_getpgid, 0), 0, "deferred_free")) + goto release; + } else { + /* + * Test the sleepable arena free path through a + * syscall test prog. + */ + ctx.skel = skel; + err = pthread_create(&thread, NULL, run_free_thread, &ctx); + if (!ASSERT_OK(err, "pthread_create")) { + skel->bss->release = 1; + goto out; + } + thread_created = true; + } + + /* Wait until the free reaches the user zap. */ + err = wait_for(&skel->bss->zap_entered); + if (!ASSERT_OK(err, "zap_entered")) + goto release; + + zap_seen = true; + + /* The range must stay allocated until the free completes. */ + run_prog(skel->progs.try_realloc, "realloc_during_free"); + ASSERT_NULL(skel->bss->realloc_ptr, "realloc_during_free"); + +release: + skel->bss->release = 1; + if (thread_created) { + ASSERT_OK(pthread_join(thread, NULL), "pthread_join"); + thread_created = false; + completed = ASSERT_OK(ctx.err, "free_run") && + ASSERT_OK(ctx.retval, "free_retval"); + } else if (skel->bss->target_tid) { + err = wait_for(&skel->bss->worker_exited); + completed = ASSERT_OK(err, "worker_exited"); + } + ASSERT_FALSE(skel->bss->timed_out, "zap_timed_out"); + + if (zap_seen && completed && + !run_prog(skel->progs.try_realloc, "realloc_after_free")) { + ASSERT_EQ((unsigned long)skel->bss->realloc_ptr, + (unsigned long)addr, "realloc_after_free"); + ASSERT_EQ(*addr, skel->rodata->new_marker, "new_marker"); + } +out: + arena_race__destroy(skel); +} + +/* + * Force a race between a faulting thread in userspace and a + * free operation on the arena. + */ +static void test_fault_free_realloc(void) +{ + struct sigaction sigbus = { .sa_handler = reader_sigbus }, old_sigbus; + struct fault_thread_ctx fault = {}; + struct arena_race *skel; + pthread_t fault_thread; + bool fault_created = false; + __u64 *addr; + __u64 expected, value; + int err, i; + + skel = setup_arena(&addr, false); + if (!skel) + return; + + skel->bss->realloc_after_free = 1; + + if (!ASSERT_OK(sigaction(SIGBUS, &sigbus, &old_sigbus), "sigaction_sigbus")) + goto out_destroy; + fault.addr = addr; + err = pthread_create(&fault_thread, NULL, fault_reader_thread, &fault); + if (!ASSERT_OK(err, "pthread_create_fault")) + goto out; + fault_created = true; + + for (i = 0; i < 1000; i++) { + LIBBPF_OPTS(bpf_test_run_opts, opts); + + /* Keep deferred frees from earlier iterations on separate slots. */ + if (i) { + skel->bss->page_idx = i; + if (run_prog(skel->progs.alloc_old, "alloc_next")) + goto out; + addr = (__u64 *)(unsigned long)skel->bss->ptr; + __atomic_store_n(&fault.addr, addr, __ATOMIC_RELEASE); + } + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.free_page), + &opts); + if (err) { + ASSERT_OK(err, "free_realloc"); + break; + } + if (opts.retval) { + ASSERT_OK(opts.retval, "free_realloc"); + break; + } + /* The reader may populate the slot first, or free may defer. */ + if (!skel->bss->realloc_ptr) + continue; + value = *addr; + expected = skel->bss->current_marker; + if (value != expected) { + ASSERT_EQ(value, expected, "marker_after_realloc"); + break; + } + } + + WRITE_ONCE(fault.stop, 1); + if (fault_created) { + ASSERT_OK(pthread_join(fault_thread, NULL), "pthread_join_fault"); + fault_created = false; + } +out: + WRITE_ONCE(fault.stop, 1); + if (fault_created) + pthread_join(fault_thread, NULL); + sigaction(SIGBUS, &old_sigbus, NULL); +out_destroy: + arena_race__destroy(skel); +} + +void serial_test_arena_race(void) +{ + cpu_set_t cpuset; + int err; + + err = sched_getaffinity(0, sizeof(cpuset), &cpuset); + if (!ASSERT_OK(err, "sched_getaffinity")) + return; + if (CPU_COUNT(&cpuset) < 2) { + printf("%s:SKIP: at least two runnable CPUs are required\n", __func__); + test__skip(); + return; + } + + if (test__start_subtest("realloc_during_free")) + test_realloc_during_free(false); + if (test__start_subtest("realloc_during_deferred_free")) + test_realloc_during_free(true); + if (test__start_subtest("fault_free_realloc")) + test_fault_free_realloc(); +} diff --git a/tools/testing/selftests/bpf/progs/arena_race.c b/tools/testing/selftests/bpf/progs/arena_race.c new file mode 100644 index 000000000000..54dbd662ca70 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/arena_race.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include +#include "bpf_experimental.h" +#include + +const volatile __u64 old_marker = 0x1111222233334444ULL; +const volatile __u64 new_marker = 0x5555666677778888ULL; + +struct { + __uint(type, BPF_MAP_TYPE_ARENA); + __uint(map_flags, BPF_F_MMAPABLE); + __uint(max_entries, 1024); +#ifdef __TARGET_ARCH_arm64 + __ulong(map_extra, 0x1ull << 32); +#else + __ulong(map_extra, 0x1ull << 44); +#endif +} arena SEC(".maps"); + +bool skip; +int page_idx; + +void __arena *ptr; +void __arena *realloc_ptr; +bool realloc_after_free; +__u64 current_marker; +__u64 marker_seq; + +int target_tid; +int pause_on_zap; +int zap_entered; +int release; +int timed_out; + +__u64 trigger_pid_tgid; +long trigger_syscall; +int deferred_free; +int worker_armed; +int worker_exited; + +static __always_inline void wait_for_release(void) +{ + while (!*(volatile int *)&release && can_loop) + ; + if (!*(volatile int *)&release) + timed_out = 1; +} + +SEC("syscall") +int alloc_old(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + char __arena *base = arena_base(&arena); + + realloc_ptr = NULL; + ptr = bpf_arena_alloc_pages(&arena, base + (page_idx + 1) * __PAGE_SIZE, 1, + NUMA_NO_NODE, 0); + if (!ptr) + return 1; + p = (__u64 __arena *)ptr; + *p = old_marker; + current_marker = old_marker; +#else + skip = true; +#endif + return 0; +} + +SEC("syscall") +int free_page(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + __u64 marker; + + if (!ptr) + return 1; + bpf_arena_free_pages(&arena, ptr, 1); + if (!realloc_after_free) + return 0; + + marker = new_marker + ++marker_seq; + realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0); + if (!realloc_ptr) + return 0; + ptr = realloc_ptr; + p = (__u64 __arena *)ptr; + *p = marker; + current_marker = marker; +#endif + return 0; +} + +SEC("syscall") +int try_realloc(void *ctx) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + __u64 __arena *p; + + realloc_ptr = bpf_arena_alloc_pages(&arena, ptr, 1, NUMA_NO_NODE, 0); + if (realloc_ptr) { + ptr = realloc_ptr; + p = (__u64 __arena *)realloc_ptr; + *p = new_marker; + current_marker = new_marker; + } +#endif + return 0; +} + +SEC("tp_btf/sys_enter") +int BPF_PROG(deferred_free_prog, struct pt_regs *regs, long id) +{ +#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM) + if (!deferred_free || bpf_get_current_pid_tgid() != trigger_pid_tgid || + id != trigger_syscall) + return 0; + + deferred_free = 0; + /* The worker can run on another CPU before the kfunc returns. */ + worker_armed = 1; + bpf_arena_free_pages(&arena, ptr, 1); +#endif + return 0; +} + +SEC("fentry/arena_free_worker") +int BPF_PROG(trace_free_worker, struct work_struct *work) +{ + if (worker_armed && !target_tid) + target_tid = (__u32)bpf_get_current_pid_tgid(); + return 0; +} + +SEC("fexit/arena_free_worker") +int BPF_PROG(trace_free_worker_ret, struct work_struct *work) +{ + if ((__u32)bpf_get_current_pid_tgid() == target_tid) + worker_exited = 1; + return 0; +} + +/* + * Pause a free while it zaps user mappings: after it cleared the kernel PTEs + * and flushed the TLB, but before it releases the range. zap_vma_range() is + * generic MM code, so it is an fentry target even where + * flush_tlb_kernel_range() is inline (arm64). + */ +SEC("?fentry/zap_vma_range") +int BPF_PROG(trace_zap, struct vm_area_struct *vma, unsigned long address, + unsigned long size) +{ + if ((__u32)bpf_get_current_pid_tgid() != target_tid) + return 0; + zap_entered = 1; + if (pause_on_zap) + wait_for_release(); + return 0; +} + +char _license[] SEC("license") = "GPL";