mirror of
https://github.com/torvalds/linux.git
synced 2026-07-28 10:09:10 +02:00
8c1e1c33fe
The BPF selftest headers include functionality that is specific to arenas and is required by libarena. Keep libarena self-contained by moving all functionality into its include/ directory. Also add libarena/include to the standard include paths for the selftests to make the moved headers easy to access by existing selftests. Some functionality is required by libarena but not strictly arena-related. We still move it to the libarena/include path, which is an upgrade from directly accessing them from the selftests/bpf directory using relative paths. A new bpf_may_goto.h file is split off of bpf_experimental.h. bpf_arena_spin_lock.h and bpf_arena_common.h are moved to libarena/include. bpf_atomic.h is also moved to libarena because it is necessary for arena spinlocks. For bpf_arena_spin_lock.h, mark the spinlock state array as __weak to define the spinlock state array in the header while also being compatible with multi-compilation unit programs. While we're at it, we remove unnecessary definitions from existing test programs. Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com> Link: https://lore.kernel.org/r/20260426190338.4615-4-emil@etsalapatis.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
143 lines
5.0 KiB
C++
143 lines
5.0 KiB
C++
// SPDX-License-Identifier: GPL-2.0
|
|
/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
|
|
#ifndef BPF_ATOMIC_H
|
|
#define BPF_ATOMIC_H
|
|
|
|
#include <vmlinux.h>
|
|
#include <bpf/bpf_helpers.h>
|
|
#include <bpf_may_goto.h>
|
|
|
|
extern bool CONFIG_X86_64 __kconfig __weak;
|
|
|
|
/*
|
|
* __unqual_typeof(x) - Declare an unqualified scalar type, leaving
|
|
* non-scalar types unchanged,
|
|
*
|
|
* Prefer C11 _Generic for better compile-times and simpler code. Note: 'char'
|
|
* is not type-compatible with 'signed char', and we define a separate case.
|
|
*
|
|
* This is copied verbatim from kernel's include/linux/compiler_types.h, but
|
|
* with default expression (for pointers) changed from (x) to (typeof(x)0).
|
|
*
|
|
* This is because LLVM has a bug where for lvalue (x), it does not get rid of
|
|
* an extra address_space qualifier, but does in case of rvalue (typeof(x)0).
|
|
* Hence, for pointers, we need to create an rvalue expression to get the
|
|
* desired type. See https://github.com/llvm/llvm-project/issues/53400.
|
|
*/
|
|
#define __scalar_type_to_expr_cases(type) \
|
|
unsigned type : (unsigned type)0, signed type : (signed type)0
|
|
|
|
#define __unqual_typeof(x) \
|
|
typeof(_Generic((x), \
|
|
char: (char)0, \
|
|
__scalar_type_to_expr_cases(char), \
|
|
__scalar_type_to_expr_cases(short), \
|
|
__scalar_type_to_expr_cases(int), \
|
|
__scalar_type_to_expr_cases(long), \
|
|
__scalar_type_to_expr_cases(long long), \
|
|
default: (typeof(x))0))
|
|
|
|
/* No-op for BPF */
|
|
#define cpu_relax() ({})
|
|
|
|
#define READ_ONCE(x) (*(volatile typeof(x) *)&(x))
|
|
|
|
#ifndef WRITE_ONCE
|
|
#define WRITE_ONCE(x, val) ((*(volatile typeof(x) *)&(x)) = (val))
|
|
#endif
|
|
|
|
#define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new)
|
|
|
|
#define try_cmpxchg(p, pold, new) \
|
|
({ \
|
|
__unqual_typeof(*(pold)) __o = *(pold); \
|
|
__unqual_typeof(*(p)) __r = cmpxchg(p, __o, new); \
|
|
if (__r != __o) \
|
|
*(pold) = __r; \
|
|
__r == __o; \
|
|
})
|
|
|
|
#define try_cmpxchg_relaxed(p, pold, new) try_cmpxchg(p, pold, new)
|
|
|
|
#define try_cmpxchg_acquire(p, pold, new) try_cmpxchg(p, pold, new)
|
|
|
|
#define smp_mb() \
|
|
({ \
|
|
volatile unsigned long __val; \
|
|
__sync_fetch_and_add(&__val, 0); \
|
|
})
|
|
|
|
#define smp_rmb() \
|
|
({ \
|
|
if (!CONFIG_X86_64) \
|
|
smp_mb(); \
|
|
else \
|
|
barrier(); \
|
|
})
|
|
|
|
#define smp_wmb() \
|
|
({ \
|
|
if (!CONFIG_X86_64) \
|
|
smp_mb(); \
|
|
else \
|
|
barrier(); \
|
|
})
|
|
|
|
/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */
|
|
#define smp_acquire__after_ctrl_dep() ({ smp_rmb(); })
|
|
|
|
#define smp_load_acquire(p) \
|
|
({ \
|
|
__unqual_typeof(*(p)) __v = READ_ONCE(*(p)); \
|
|
if (!CONFIG_X86_64) \
|
|
smp_mb(); \
|
|
barrier(); \
|
|
__v; \
|
|
})
|
|
|
|
#define smp_store_release(p, val) \
|
|
({ \
|
|
if (!CONFIG_X86_64) \
|
|
smp_mb(); \
|
|
barrier(); \
|
|
WRITE_ONCE(*(p), val); \
|
|
})
|
|
|
|
#define smp_cond_load_relaxed_label(p, cond_expr, label) \
|
|
({ \
|
|
typeof(p) __ptr = (p); \
|
|
__unqual_typeof(*(p)) VAL; \
|
|
for (;;) { \
|
|
VAL = (__unqual_typeof(*(p)))READ_ONCE(*__ptr); \
|
|
if (cond_expr) \
|
|
break; \
|
|
cond_break_label(label); \
|
|
cpu_relax(); \
|
|
} \
|
|
(typeof(*(p)))VAL; \
|
|
})
|
|
|
|
#define smp_cond_load_acquire_label(p, cond_expr, label) \
|
|
({ \
|
|
__unqual_typeof(*p) __val = \
|
|
smp_cond_load_relaxed_label(p, cond_expr, label); \
|
|
smp_acquire__after_ctrl_dep(); \
|
|
(typeof(*(p)))__val; \
|
|
})
|
|
|
|
#define atomic_read(p) READ_ONCE((p)->counter)
|
|
|
|
#define atomic_cond_read_relaxed_label(p, cond_expr, label) \
|
|
smp_cond_load_relaxed_label(&(p)->counter, cond_expr, label)
|
|
|
|
#define atomic_cond_read_acquire_label(p, cond_expr, label) \
|
|
smp_cond_load_acquire_label(&(p)->counter, cond_expr, label)
|
|
|
|
#define atomic_try_cmpxchg_relaxed(p, pold, new) \
|
|
try_cmpxchg_relaxed(&(p)->counter, pold, new)
|
|
|
|
#define atomic_try_cmpxchg_acquire(p, pold, new) \
|
|
try_cmpxchg_acquire(&(p)->counter, pold, new)
|
|
|
|
#endif /* BPF_ATOMIC_H */
|