github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-01 11:03:43 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'bpf-arena-followups'

Browse Source 
Alexei Starovoitov says:

====================
bpf: arena followups.

From: Alexei Starovoitov <ast@kernel.org>

A set of follow ups to clean up bpf_arena and adjust to the latest LLVM.
====================

Link: https://lore.kernel.org/r/20240315021834.62988-1-alexei.starovoitov@gmail.com
Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
This commit is contained in:
Andrii Nakryiko 2024-03-15 14:09:05 -07:00
commit 0740b6427e
11 changed files with 109 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
kernel/bpf/arena.c
View File

@ -38,7 +38,7 @@
/* number of bytes addressable by LDX/STX insn with 16-bit 'off' field */
#define GUARD_SZ (1ull << sizeof(((struct bpf_insn *)0)->off) * 8)
#define KERN_VM_SZ ((1ull << 32) + GUARD_SZ)
#define KERN_VM_SZ (SZ_4G + GUARD_SZ)
struct bpf_arena {
struct bpf_map map;
@ -110,7 +110,7 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr)
return ERR_PTR(-EINVAL);
vm_range = (u64)attr->max_entries * PAGE_SIZE;
if (vm_range > (1ull << 32))
if (vm_range > SZ_4G)
return ERR_PTR(-E2BIG);
if ((attr->map_extra >> 32) != ((attr->map_extra + vm_range - 1) >> 32))
@ -301,7 +301,7 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad
if (pgoff)
return -EINVAL;
if (len > (1ull << 32))
if (len > SZ_4G)
return -E2BIG;
/* if user_vm_start was specified at arena creation time */
@ -322,7 +322,7 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad
if (WARN_ON_ONCE(arena->user_vm_start))
/* checks at map creation time should prevent this */
return -EFAULT;
return round_up(ret, 1ull << 32);
return round_up(ret, SZ_4G);
}
static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
@ -346,7 +346,7 @@ static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
return -EBUSY;
/* Earlier checks should prevent this */
if (WARN_ON_ONCE(vma->vm_end - vma->vm_start > (1ull << 32) || vma->vm_pgoff))
if (WARN_ON_ONCE(vma->vm_end - vma->vm_start > SZ_4G || vma->vm_pgoff))
return -EFAULT;
if (remember_vma(arena, vma))
@ -420,7 +420,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
if (uaddr & ~PAGE_MASK)
return 0;
pgoff = compute_pgoff(arena, uaddr);
if (pgoff + page_cnt > page_cnt_max)
if (pgoff > page_cnt_max - page_cnt)
/* requested address will be outside of user VMA */
return 0;
}
@ -447,7 +447,13 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt
goto out;
uaddr32 = (u32)(arena->user_vm_start + pgoff * PAGE_SIZE);
/* Earlier checks make sure that uaddr32 + page_cnt * PAGE_SIZE will not overflow 32-bit */
/* Earlier checks made sure that uaddr32 + page_cnt * PAGE_SIZE - 1
* will not overflow 32-bit. Lower 32-bit need to represent
* contiguous user address range.
* Map these pages at kern_vm_start base.
* kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE - 1 can overflow
* lower 32-bit and it's ok.
*/
ret = vm_area_map_pages(arena->kern_vm, kern_vm_start + uaddr32,
kern_vm_start + uaddr32 + page_cnt * PAGE_SIZE, pages);
if (ret) {
@ -510,6 +516,11 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt)
if (!page)
continue;
if (page_cnt == 1 && page_mapped(page)) /* mapped by some user process */
/* Optimization for the common case of page_cnt==1:
* If page wasn't mapped into some user vma there
* is no need to call zap_pages which is slow. When
* page_cnt is big it's faster to do the batched zap.
*/
zap_pages(arena, full_uaddr, 1);
vm_area_unmap_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE);
__free_page(page);

2
tools/bpf/bpftool/gen.c
View File

@ -121,7 +121,7 @@ static bool get_datasec_ident(const char *sec_name, char *buf, size_t buf_sz)
int i, n;
/* recognize hard coded LLVM section name */
if (strcmp(sec_name, ".arena.1") == 0) {
if (strcmp(sec_name, ".addr_space.1") == 0) {
/* this is the name to use in skeleton */
snprintf(buf, buf_sz, "arena");
return true;

2
tools/lib/bpf/libbpf.c
View File

@ -498,7 +498,7 @@ struct bpf_struct_ops {
#define KSYMS_SEC ".ksyms"
#define STRUCT_OPS_SEC ".struct_ops"
#define STRUCT_OPS_LINK_SEC ".struct_ops.link"
#define ARENA_SEC ".arena.1"
#define ARENA_SEC ".addr_space.1"
enum libbpf_map_type {
LIBBPF_MAP_UNSPEC,

2
tools/testing/selftests/bpf/bpf_arena_common.h
View File

@ -32,7 +32,7 @@
*/
#endif
#if defined(__BPF_FEATURE_ARENA_CAST) && !defined(BPF_ARENA_FORCE_ASM)
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) && !defined(BPF_ARENA_FORCE_ASM)
#define __arena __attribute__((address_space(1)))
#define cast_kern(ptr) /* nop for bpf prog. emitted by LLVM */
#define cast_user(ptr) /* nop for bpf prog. emitted by LLVM */

8
tools/testing/selftests/bpf/prog_tests/arena_htab.c
View File

@ -3,12 +3,14 @@
#include <test_progs.h>
#include <sys/mman.h>
#include <network_helpers.h>
#include <sys/user.h>
#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */
#include <unistd.h>
#define PAGE_SIZE getpagesize()
#endif
#include "arena_htab_asm.skel.h"
#include "arena_htab.skel.h"
#define PAGE_SIZE 4096
#include "bpf_arena_htab.h"
static void test_arena_htab_common(struct htab *htab)

7
tools/testing/selftests/bpf/prog_tests/arena_list.c
View File

@ -3,8 +3,11 @@
#include <test_progs.h>
#include <sys/mman.h>
#include <network_helpers.h>
#define PAGE_SIZE 4096
#include <sys/user.h>
#ifndef PAGE_SIZE /* on some archs it comes in sys/user.h */
#include <unistd.h>
#define PAGE_SIZE getpagesize()
#endif
#include "bpf_arena_list.h"
#include "arena_list.skel.h"

2
tools/testing/selftests/bpf/prog_tests/verifier.c
View File

@ -5,6 +5,7 @@
#include "cap_helpers.h"
#include "verifier_and.skel.h"
#include "verifier_arena.skel.h"
#include "verifier_arena_large.skel.h"
#include "verifier_array_access.skel.h"
#include "verifier_basic_stack.skel.h"
#include "verifier_bitfield_write.skel.h"
@ -120,6 +121,7 @@ static void run_tests_aux(const char *skel_name,
void test_verifier_and(void) { RUN(verifier_and); }
void test_verifier_arena(void) { RUN(verifier_arena); }
void test_verifier_arena_large(void) { RUN(verifier_arena_large); }
void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); }
void test_verifier_bitfield_write(void) { RUN(verifier_bitfield_write); }
void test_verifier_bounds(void) { RUN(verifier_bounds); }

2
tools/testing/selftests/bpf/progs/arena_htab.c
View File

@ -22,7 +22,7 @@ int zero = 0;
SEC("syscall")
int arena_htab_llvm(void *ctx)
{
#if defined(__BPF_FEATURE_ARENA_CAST) || defined(BPF_ARENA_FORCE_ASM)
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) || defined(BPF_ARENA_FORCE_ASM)
struct htab __arena *htab;
__u64 i;

10
tools/testing/selftests/bpf/progs/arena_list.c
View File

@ -30,13 +30,13 @@ int list_sum;
int cnt;
bool skip = false;
#ifdef __BPF_FEATURE_ARENA_CAST
#ifdef __BPF_FEATURE_ADDR_SPACE_CAST
long __arena arena_sum;
int __arena test_val = 1;
struct arena_list_head __arena global_head;
#else
long arena_sum SEC(".arena.1");
int test_val SEC(".arena.1");
long arena_sum SEC(".addr_space.1");
int test_val SEC(".addr_space.1");
#endif
int zero;
@ -44,7 +44,7 @@ int zero;
SEC("syscall")
int arena_list_add(void *ctx)
{
#ifdef __BPF_FEATURE_ARENA_CAST
#ifdef __BPF_FEATURE_ADDR_SPACE_CAST
__u64 i;
list_head = &global_head;
@ -66,7 +66,7 @@ int arena_list_add(void *ctx)
SEC("syscall")
int arena_list_del(void *ctx)
{
#ifdef __BPF_FEATURE_ARENA_CAST
#ifdef __BPF_FEATURE_ADDR_SPACE_CAST
struct elem __arena *n;
int sum = 0;

4
tools/testing/selftests/bpf/progs/verifier_arena.c
View File

@ -19,7 +19,7 @@ SEC("syscall")
__success __retval(0)
int basic_alloc1(void *ctx)
{
#if defined(__BPF_FEATURE_ARENA_CAST)
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
volatile int __arena *page1, *page2, *no_page, *page3;
page1 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
@ -58,7 +58,7 @@ SEC("syscall")
__success __retval(0)
int basic_alloc2(void *ctx)
{
#if defined(__BPF_FEATURE_ARENA_CAST)
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
volatile char __arena *page1, *page2, *page3, *page4;
page1 = bpf_arena_alloc_pages(&arena, NULL, 2, NUMA_NO_NODE, 0);

68
tools/testing/selftests/bpf/progs/verifier_arena_large.c Normal file
View File

@ -0,0 +1,68 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
#include <vmlinux.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "bpf_misc.h"
#include "bpf_experimental.h"
#include "bpf_arena_common.h"
#define ARENA_SIZE (1ull << 32)
struct {
__uint(type, BPF_MAP_TYPE_ARENA);
__uint(map_flags, BPF_F_MMAPABLE);
__uint(max_entries, ARENA_SIZE / PAGE_SIZE);
} arena SEC(".maps");
SEC("syscall")
__success __retval(0)
int big_alloc1(void *ctx)
{
#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
volatile char __arena *page1, *page2, *no_page, *page3;
void __arena *base;
page1 = base = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
if (!page1)
return 1;
*page1 = 1;
page2 = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE,
1, NUMA_NO_NODE, 0);
if (!page2)
return 2;
*page2 = 2;
no_page = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE,
1, NUMA_NO_NODE, 0);
if (no_page)
return 3;
if (*page1 != 1)
return 4;
if (*page2 != 2)
return 5;
bpf_arena_free_pages(&arena, (void __arena *)page1, 1);
if (*page2 != 2)
return 6;
if (*page1 != 0) /* use-after-free should return 0 */
return 7;
page3 = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
if (!page3)
return 8;
*page3 = 3;
if (page1 != page3)
return 9;
if (*page2 != 2)
return 10;
if (*(page1 + PAGE_SIZE) != 0)
return 11;
if (*(page1 - PAGE_SIZE) != 0)
return 12;
if (*(page2 + PAGE_SIZE) != 0)
return 13;
if (*(page2 - PAGE_SIZE) != 0)
return 14;
#endif
return 0;
}
char _license[] SEC("license") = "GPL";