github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-28 12:02:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'bpf-arm64-__arena-kfunc-and-struct_ops-arguments'

Browse Source 
Puranjay Mohan says:

====================
bpf, arm64: __arena kfunc and struct_ops arguments

The x86-64 JIT recently gained support for the __arena and
__arena__nullable argument suffixes on kfuncs and struct_ops stubs. This
adds the arm64 side and flips bpf_jit_supports_arena_args() on, so the
verifier stops rejecting these programs on arm64.

Patch 1 is an independent fix. save_args() reads stack-passed arguments
at FP + 32, which only holds when the trampoline is entered through the
fentry call and two frame records are pushed. A struct_ops trampoline is
entered via blr and pushes one frame fewer, so its stack arguments start
at FP + 16 and every one of them was read two slots off. No struct_ops
member passed arguments on the stack until the test added by commit
2d4de9a493, which is why this went unnoticed. It carries a Fixes tag
and can be taken separately; note that the test covering it only runs on
arm64 once the rest of this series lands.

Patch 2 adds an ADD/SUB (extended register) encoder to the insn library,
so the JIT can zero-extend and add in one instruction.

Patches 3 and 4 are the JIT work. A kfunc argument is rebased onto the
arena base at the call site:

        add     xN, x28, wN, uxtw

and a nullable one skips the add so NULL stays NULL:

        mov     wN, wN
        cbz     wN, 1f
        add     xN, x28, wN, uxtw
1:

A struct_ops callback converts in the other direction, in the trampoline
while saving arguments into the BPF ctx, with the low half of the arena
base kept in x11:

        sub     w10, wsrc, w11
        str     x10, [sp, #slot]

Patches 5 and 6 add arm64 JIT-sequence assertions and drop the x86-64
gating from the existing arena argument tests. Patch 7 is arch-neutral:
it adds a struct_ops member whose first argument is a 16-byte struct
passed by value, so the arena pointer does not land at the ctx slot its
argument index suggests. Nothing covered that before, and it is the case
patch 4 has to get right.

Changelog:
V1: https://lore.kernel.org/bpf/20260810190922.3408757-1-puranjay@kernel.org/
Changes in v2:
- patch 2: fix the decode masks for the new extended-register predicates,
  0x7F200000 -> 0x7FE00000. opt in bits 23:22 is part of the opcode here
  rather than a shift type, and any value other than 00 is unallocated
  (Xu Kuohai). Also noted in the commit message. No functional change: the
  masks only feed aarch64_insn_is_*_ext(), which has no in-tree callers,
  while the encoder uses aarch64_insn_get_*_ext_value().
- patch 4: comment why the conversion in the stack-argument loop is not
  guarded by for_call_origin (Xu Kuohai).
- collect Reviewed-by/Acked-by from Xu Kuohai.
- rebase onto current bpf-next.
====================

Link: https://patch.msgid.link/20260813190356.335181-1-puranjay@kernel.org
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
This commit is contained in:
Kumar Kartikeya Dwivedi 2026-08-14 05:55:05 +02:00
commit 409a9bda04
No known key found for this signature in database
GPG Key ID: 472D377B63542F83
12 changed files with 309 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Documentation/bpf/kfuncs.rst
View File

@ -301,9 +301,9 @@ An example is given below::
}
Calling such a kfunc requires the program to use an arena map and a JIT with
arena argument support (currently x86-64); verification fails otherwise. The
program can pass any value without compromising the kernel. A value that does
not point into the arena is a program bug.
arena argument support (currently x86-64 and arm64); verification fails
otherwise. The program can pass any value without compromising the kernel. A
value that does not point into the arena is a program bug.
The suffixes have the same meaning on the arguments of struct_ops stub
functions, with the conversion running in the opposite direction. The

23
arch/arm64/include/asm/insn.h
View File

@ -205,6 +205,18 @@ enum aarch64_insn_adsb_type {
AARCH64_INSN_ADSB_SUB_SETFLAGS
};
/* option field of add/sub (extended register) */
enum aarch64_insn_extend_type {
AARCH64_INSN_EXTEND_UXTB,
AARCH64_INSN_EXTEND_UXTH,
AARCH64_INSN_EXTEND_UXTW,
AARCH64_INSN_EXTEND_UXTX,
AARCH64_INSN_EXTEND_SXTB,
AARCH64_INSN_EXTEND_SXTH,
AARCH64_INSN_EXTEND_SXTW,
AARCH64_INSN_EXTEND_SXTX,
};
enum aarch64_insn_movewide_type {
AARCH64_INSN_MOVEWIDE_ZERO,
AARCH64_INSN_MOVEWIDE_KEEP,
@ -378,6 +390,10 @@ __AARCH64_INSN_FUNCS(add, 0x7F200000, 0x0B000000)
__AARCH64_INSN_FUNCS(adds, 0x7F200000, 0x2B000000)
__AARCH64_INSN_FUNCS(sub, 0x7F200000, 0x4B000000)
__AARCH64_INSN_FUNCS(subs, 0x7F200000, 0x6B000000)
__AARCH64_INSN_FUNCS(add_ext, 0x7FE00000, 0x0B200000)
__AARCH64_INSN_FUNCS(adds_ext, 0x7FE00000, 0x2B200000)
__AARCH64_INSN_FUNCS(sub_ext, 0x7FE00000, 0x4B200000)
__AARCH64_INSN_FUNCS(subs_ext, 0x7FE00000, 0x6B200000)
__AARCH64_INSN_FUNCS(madd, 0x7FE08000, 0x1B000000)
__AARCH64_INSN_FUNCS(msub, 0x7FE08000, 0x1B008000)
__AARCH64_INSN_FUNCS(udiv, 0x7FE0FC00, 0x1AC00800)
@ -637,6 +653,13 @@ u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
int shift,
enum aarch64_insn_variant variant,
enum aarch64_insn_adsb_type type);
u32 aarch64_insn_gen_add_sub_extended_reg(enum aarch64_insn_register dst,
enum aarch64_insn_register src,
enum aarch64_insn_register reg,
enum aarch64_insn_extend_type extend,
int shift,
enum aarch64_insn_variant variant,
enum aarch64_insn_adsb_type type);
u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
enum aarch64_insn_register src,
enum aarch64_insn_variant variant,

60
arch/arm64/lib/insn.c
View File

@ -986,6 +986,66 @@ u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
}
/*
* Unlike the shifted-register form, register 31 is not XZR everywhere here:
* it encodes SP for @src, and for @dst too unless @type sets the flags. Only
* @reg keeps the XZR meaning.
*/
u32 aarch64_insn_gen_add_sub_extended_reg(enum aarch64_insn_register dst,
enum aarch64_insn_register src,
enum aarch64_insn_register reg,
enum aarch64_insn_extend_type extend,
int shift,
enum aarch64_insn_variant variant,
enum aarch64_insn_adsb_type type)
{
u32 insn;
switch (type) {
case AARCH64_INSN_ADSB_ADD:
insn = aarch64_insn_get_add_ext_value();
break;
case AARCH64_INSN_ADSB_SUB:
insn = aarch64_insn_get_sub_ext_value();
break;
case AARCH64_INSN_ADSB_ADD_SETFLAGS:
insn = aarch64_insn_get_adds_ext_value();
break;
case AARCH64_INSN_ADSB_SUB_SETFLAGS:
insn = aarch64_insn_get_subs_ext_value();
break;
default:
pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
return AARCH64_BREAK_FAULT;
}
switch (variant) {
case AARCH64_INSN_VARIANT_32BIT:
break;
case AARCH64_INSN_VARIANT_64BIT:
insn |= AARCH64_INSN_SF_BIT;
break;
default:
pr_err("%s: unknown variant encoding %d\n", __func__, variant);
return AARCH64_BREAK_FAULT;
}
if (shift < 0 || shift > 4) {
pr_err("%s: invalid shift encoding %d\n", __func__, shift);
return AARCH64_BREAK_FAULT;
}
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
/* option in bits [15:13] and imm3 in [12:10] together fill IMM_6 */
return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn,
(extend << 3) | shift);
}
u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
enum aarch64_insn_register src,
enum aarch64_insn_variant variant,

11
arch/arm64/net/bpf_jit.h
View File

@ -243,6 +243,17 @@
/* Rn - Rm; set condition flags */
#define A64_CMP(sf, Rn, Rm) A64_SUBS(sf, A64_ZR, Rn, Rm)
/* Add/subtract (extended register) */
#define A64_ADDSUB_EREG(sf, Rd, Rn, Rm, ext, shift, type) \
aarch64_insn_gen_add_sub_extended_reg(Rd, Rn, Rm, \
AARCH64_INSN_EXTEND_##ext, shift, A64_VARIANT(sf), \
AARCH64_INSN_ADSB_##type)
/* Rd = Rn + (EXT(Rm) << shift) */
#define A64_ADD_EXT(sf, Rd, Rn, Rm, ext, shift) \
A64_ADDSUB_EREG(sf, Rd, Rn, Rm, ext, shift, ADD)
/* Rd = Rn + (u32)Rm */
#define A64_ADD_UXTW(Rd, Rn, Rm) A64_ADD_EXT(1, Rd, Rn, Rm, UXTW, 0)
/* Data-processing (1 source) */
#define A64_DATA1(sf, Rd, Rn, type) aarch64_insn_gen_data1(Rd, Rn, \
A64_VARIANT(sf), AARCH64_INSN_DATA1_##type)

149
arch/arm64/net/bpf_jit_comp.c
View File

@ -1256,6 +1256,43 @@ static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct
}
}
/*
* Rebase the __arena args of a kfunc call to arena kernel addresses,
* xN = kern_vm_start + (u32)xN, with the arena base register holding
* kern_vm_start. A nullable arg preserves NULL by skipping the add, tested
* on the truncated value as arena NULL is offset 0.
*/
static int emit_kfunc_arena_args(struct jit_ctx *ctx, const struct bpf_insn *insn)
{
const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
const struct btf_func_model *fm;
int i;
fm = bpf_jit_find_kfunc_model(ctx->prog, insn);
if (!fm)
return -EINVAL;
for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) {
const u8 reg = bpf2a64[BPF_REG_1 + i];
u8 flags = fm->arg_flags[i];
if (!(flags & BTF_FMODEL_ARENA_ARG))
continue;
if (WARN_ON_ONCE(!ctx->arena_vm_start))
return -EINVAL;
if (flags & BTF_FMODEL_NULLABLE_ARG) {
/* 32-bit mov clears the upper 32 bits */
emit(A64_MOV(0, reg, reg), ctx);
/* skip the add so that NULL stays NULL */
emit(A64_CBZ(0, reg, 2), ctx);
}
emit(A64_ADD_UXTW(reg, arena_vm_base, reg), ctx);
}
return 0;
}
/* JITs an eBPF instruction.
* Returns:
* 0 - successfully JITed an 8-byte eBPF instruction.
@ -1678,6 +1715,11 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
&func_addr, &func_addr_fixed);
if (ret < 0)
return ret;
if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
ret = emit_kfunc_arena_args(ctx, insn);
if (ret < 0)
return ret;
}
emit_call(func_addr, ctx);
/*
* Call to arch_bpf_timed_may_goto() is emitted by the
@ -2351,6 +2393,11 @@ bool bpf_jit_supports_stack_args(void)
return true;
}
bool bpf_jit_supports_arena_args(void)
{
return true;
}
void *bpf_arch_text_copy(void *dst, void *src, size_t len)
{
if (!aarch64_insn_copy(dst, src, len))
@ -2524,34 +2571,76 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes)
}
}
static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
const struct btf_func_model *m,
const struct arg_aux *a,
bool for_call_origin)
/*
* Convert an arena kernel address into the arena pointer form on its way into
* the BPF ctx, dst = (u32)(src - kern_vm_start), with @base_lo holding the low
* 32 bits of kern_vm_start. A nullable arg preserves NULL, tested on the full
* 64-bit kernel pointer. The 32-bit subtraction both truncates and clears the
* upper half, so the stored value satisfies the JIT invariant for arena
* pointer registers.
*/
static void emit_arena_arg_conv(struct jit_ctx *ctx, u8 dst, u8 src, bool nullable, u8 base_lo)
{
if (nullable) {
if (dst != src)
emit(A64_MOV(1, dst, src), ctx);
/* skip the subtraction so that NULL stays NULL */
emit(A64_CBZ(1, dst, 2), ctx);
src = dst;
}
emit(A64_SUB(0, dst, src, base_lo), ctx);
}
static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
const struct btf_func_model *m, const struct arg_aux *a,
bool for_call_origin, bool is_struct_ops, u64 arena_base)
{
int i;
int reg;
int doff;
int soff;
int slots;
u8 tmp = bpf2a64[TMP_REG_1];
u8 base_lo = bpf2a64[TMP_REG_2];
int i, reg, doff, soff, slots;
/* only the low 32 bits of the base take part in the subtraction */
if (arena_base)
emit_a64_mov_i(0, base_lo, (s32)(u32)arena_base, ctx);
/* store arguments to the stack for the bpf program, or restore
* arguments from stack for the original function
*/
for (reg = 0; reg < a->regs_for_args; reg++) {
emit(for_call_origin ?
A64_LDR64I(reg, A64_SP, bargs_off) :
A64_STR64I(reg, A64_SP, bargs_off),
ctx);
bargs_off += 8;
for (i = 0, reg = 0; i < a->args_in_regs; i++) {
bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG);
bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG;
slots = (m->arg_size[i] + 7) / 8;
while (slots-- > 0) {
if (for_call_origin) {
emit(A64_LDR64I(reg, A64_SP, bargs_off), ctx);
} else if (arena_arg) {
emit_arena_arg_conv(ctx, tmp, reg, nullable, base_lo);
emit(A64_STR64I(tmp, A64_SP, bargs_off), ctx);
} else {
emit(A64_STR64I(reg, A64_SP, bargs_off), ctx);
}
reg++;
bargs_off += 8;
}
}
soff = 32; /* on stack arguments start from FP + 32 */
/*
* On-stack arguments start above the frame(s) pushed by the trampoline
* prologue. Entered through the fentry call from a traced function, the
* prologue saves both the parent (FP/x9) and the traced function
* (FP/LR) frames, so the arguments start at FP + 32. A struct_ops
* callback is called indirectly and only the FP/LR frame is saved, so
* they start at FP + 16.
*/
soff = is_struct_ops ? 16 : 32;
doff = (for_call_origin ? oargs_off : bargs_off);
/* save on stack arguments */
for (i = a->args_in_regs; i < m->nr_args; i++) {
bool arena_arg = arena_base && (m->arg_flags[i] & BTF_FMODEL_ARENA_ARG);
bool nullable = m->arg_flags[i] & BTF_FMODEL_NULLABLE_ARG;
slots = (m->arg_size[i] + 7) / 8;
/* verifier ensures arg_size <= 16, so slots equals 1 or 2 */
while (slots-- > 0) {
@ -2561,6 +2650,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off,
*/
if (slots == 0 && !for_call_origin)
clear_garbage(ctx, tmp, m->arg_size[i] % 8);
/*
* No guard on for_call_origin here: only the indirect
* trampoline is given a base, and it never calls the
* original function, so arguments are never converted
* on their way back out to it. See the WARN_ON_ONCE()
* in prepare_trampoline().
*/
if (arena_arg)
emit_arena_arg_conv(ctx, tmp, tmp, nullable, base_lo);
emit(A64_STR64I(tmp, A64_SP, doff), ctx);
soff += 8;
doff += 8;
@ -2620,8 +2718,21 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
bool is_struct_ops = is_struct_ops_tramp(fentry);
int cookie_off, cookie_cnt, cookie_bargs_off;
int fsession_cnt = bpf_fsession_cnt(tnodes);
u64 arena_base;
u64 func_meta;
/*
* F_INDIRECT is only compatible with F_RET_FENTRY_RET, it is explicitly
* incompatible with F_CALL_ORIG | F_SKIP_FRAME | F_IP_ARG because
* @func_addr. Arena conversion relies on this: bpf_tramp_arena_base()
* only returns a base for the indirect trampoline, which therefore
* never calls the original function with converted arguments.
*/
WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) &&
(flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET)));
arena_base = bpf_tramp_arena_base(m, tnodes, flags);
/* trampoline stack layout:
* [ parent ip ]
* [ FP ]
@ -2737,7 +2848,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
store_func_meta(ctx, func_meta, func_meta_off);
/* save args for bpf */
save_args(ctx, bargs_off, oargs_off, m, a, false);
save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops, arena_base);
/* save callee saved registers */
emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx);
@ -2785,8 +2896,8 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im,
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
/* save args for original func */
save_args(ctx, bargs_off, oargs_off, m, a, true);
/* the original func takes kernel addresses, never converted ones */
save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops, 0);
/* call original func */
emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx);
emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx);

10
tools/testing/selftests/bpf/prog_tests/test_struct_ops_arena.c
View File

@ -6,7 +6,7 @@
#include "struct_ops_arena_attach.skel.h"
#include "struct_ops_arena_fail.skel.h"
#if defined(__x86_64__)
#if defined(__x86_64__) || defined(__aarch64__)
/*
* Attach callbacks with __arena and __arena__nullable arguments and drive
* them through the bpf_testmod_ops3_call_test_arena*() kfuncs.
@ -111,11 +111,11 @@ static void arena_arg_attach(void)
void serial_test_struct_ops_arena(void)
{
/*
* Arena struct_ops arguments need JIT support, currently x86-64 only.
* Elsewhere verification fails with "JIT does not support arena
* arguments", so the programs cannot even load.
* Arena struct_ops arguments need JIT support, currently x86-64 and
* arm64 only. Elsewhere verification fails with "JIT does not support
* arena arguments", so the programs cannot even load.
*/
#if defined(__x86_64__)
#if defined(__x86_64__) || defined(__aarch64__)
if (test__start_subtest("arena_arg"))
arena_arg();
if (test__start_subtest("arena_arg_fail"))

9
tools/testing/selftests/bpf/progs/arena_kfunc.c
View File

@ -27,6 +27,7 @@ volatile u64 stash;
SEC("syscall")
__arch_x86_64
__arch_arm64
__success __retval(0)
int arena_arg_forms(void *ctx)
{
@ -70,6 +71,7 @@ int arena_arg_forms(void *ctx)
*/
SEC("syscall")
__arch_x86_64
__arch_arm64
__success __retval(0)
int arena_arg_rebase(void *ctx)
{
@ -111,6 +113,7 @@ int arena_arg_rebase(void *ctx)
SEC("syscall")
__arch_x86_64
__arch_arm64
__success __retval(0)
int arena_args5(void *ctx)
{
@ -142,6 +145,7 @@ int arena_args5(void *ctx)
SEC("syscall")
__arch_x86_64
__arch_arm64
__success __retval(0)
int arena_arg_mixed(void *ctx)
{
@ -169,6 +173,7 @@ int arena_arg_mixed(void *ctx)
/* kernel-side faults on unpopulated pages recover via the scratch page */
SEC("syscall")
__arch_x86_64
__arch_arm64
__success __retval(0)
int arena_arg_unpopulated(void *ctx)
{
@ -189,6 +194,7 @@ int arena_arg_unpopulated(void *ctx)
SEC("syscall")
__arch_x86_64
__arch_arm64
__failure __msg("arena pointer requires a program with an associated arena")
int arena_arg_no_arena(void *ctx)
{
@ -198,6 +204,7 @@ int arena_arg_no_arena(void *ctx)
SEC("syscall")
__arch_x86_64
__arch_arm64
__failure __msg("is not a pointer to arena or scalar")
int arena_arg_bad_reg(void *ctx)
{
@ -213,6 +220,7 @@ int arena_arg_bad_reg(void *ctx)
defined(__BPF_FEATURE_STACK_ARGUMENT)
SEC("syscall")
__arch_x86_64
__arch_arm64
__failure __msg("arena pointer cannot be a stack argument")
int arena_arg_stack(void *ctx)
{
@ -223,6 +231,7 @@ int arena_arg_stack(void *ctx)
#else
SEC("syscall")
__arch_x86_64
__arch_arm64
__description("arena_arg_stack: not supported, dummy test")
__success
int arena_arg_stack(void *ctx)

20
tools/testing/selftests/bpf/progs/arena_kfunc_jit.c
View File

@ -32,6 +32,10 @@ __jited(" movl %edi, %edi")
__jited(" addq %r12, %rdi")
__jited("...")
__jited(" callq {{.*}}")
__arch_arm64
__jited("...")
__jited(" add x0, x28, w0, uxtw")
__jited(" {{(bl|mov) .*}}")
__success
int arena_arg_jit_rebase(void *ctx)
{
@ -48,6 +52,12 @@ __jited(" testl %edi, %edi")
__jited(" je L0")
__jited(" addq %r12, %rdi")
__jited("L0: callq {{.*}}")
__arch_arm64
__jited("...")
__jited(" mov w0, w0")
__jited(" cbz w0, L0")
__jited(" add x0, x28, w0, uxtw")
__jited("L0: {{.*}}")
__success
int arena_arg_jit_nullable(void *ctx)
{
@ -72,6 +82,16 @@ __jited(" testl %r8d, %r8d")
__jited(" je L0")
__jited(" addq %r12, %r8")
__jited("L0: callq {{.*}}")
__arch_arm64
__jited("...")
__jited(" add x0, x28, w0, uxtw")
__jited(" add x1, x28, w1, uxtw")
__jited(" add x2, x28, w2, uxtw")
__jited(" add x3, x28, w3, uxtw")
__jited(" mov w4, w4")
__jited(" cbz w4, L0")
__jited(" add x4, x28, w4, uxtw")
__jited("L0: {{.*}}")
__success
int arena_arg_jit_args5(void *ctx)
{

24
tools/testing/selftests/bpf/progs/struct_ops_arena.c
View File

@ -59,11 +59,28 @@ int test_arena_stack_cb(unsigned long long *ctx)
return 0;
}
SEC("struct_ops/test_arena_multislot")
int test_arena_multislot_cb(unsigned long long *ctx)
{
u64 __arena *ptr = (u64 __arena *)ctx[2];
arena_touch++;
/*
* The 16-byte struct occupies ctx[0] and ctx[1], so @ptr is argument
* one but slot two. Getting that wrong hands the callback a scalar.
*/
if (ctx[0] != 11 || ctx[1] != 22)
return 0xbad;
*ptr += 1;
return 0;
}
SEC(".struct_ops.link")
struct bpf_testmod_ops3 testmod_arena = {
.test_arena = (void *)test_arena_cb,
.test_arena_nullable = (void *)test_arena_nullable_cb,
.test_arena_stack = (void *)test_arena_stack_cb,
.test_arena_multislot = (void *)test_arena_multislot_cb,
};
SEC("syscall")
@ -109,6 +126,13 @@ int trigger(void *ctx)
if (*val != 44)
return 9;
/* a multi-slot arg precedes the arena pointer here */
ret = bpf_testmod_ops3_call_test_arena_multislot((u64 *)val);
if (ret)
return 10;
if (*val != 45)
return 11;
bpf_arena_free_pages(&arena, (void __arena *)val, 1);
#endif
return 0;

15
tools/testing/selftests/bpf/test_kmods/bpf_testmod.c
View File

@ -402,12 +402,19 @@ static int bpf_testmod_ops3__test_arena_stack(u64 a, u64 b, u64 c, u64 d,
return 0;
}
static int bpf_testmod_ops3__test_arena_multislot(struct bpf_testmod_arena_pair p,
u64 *ptr__arena)
{
return 0;
}
static struct bpf_testmod_ops3 __bpf_testmod_ops3 = {
.test_1 = bpf_testmod_test_3,
.test_2 = bpf_testmod_test_4,
.test_arena = bpf_testmod_ops3__test_arena,
.test_arena_nullable = bpf_testmod_ops3__test_arena_nullable,
.test_arena_stack = bpf_testmod_ops3__test_arena_stack,
.test_arena_multislot = bpf_testmod_ops3__test_arena_multislot,
};
static void bpf_testmod_test_struct_ops3(void)
@ -441,6 +448,13 @@ __bpf_kfunc int bpf_testmod_ops3_call_test_arena_stack(u64 *ptr__arena)
return st_ops3->test_arena_stack(1, 2, 3, 4, 5, 6, 7, 8, ptr__arena);
}
__bpf_kfunc int bpf_testmod_ops3_call_test_arena_multislot(u64 *ptr__arena)
{
struct bpf_testmod_arena_pair p = { .a = 11, .b = 22 };
return st_ops3->test_arena_multislot(p, ptr__arena);
}
struct bpf_testmod_btf_type_tag_1 {
int a;
};
@ -852,6 +866,7 @@ BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_2)
BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena)
BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_nullable)
BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_stack)
BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_arena_multislot)
BTF_ID_FLAGS(func, bpf_kfunc_get_default_trusted_ptr_test);
BTF_ID_FLAGS(func, bpf_kfunc_put_default_trusted_ptr_test);
BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids)

8
tools/testing/selftests/bpf/test_kmods/bpf_testmod.h
View File

@ -103,6 +103,12 @@ struct bpf_testmod_ops2 {
int (*test_1)(void);
};
/* 16 bytes, so it takes two argument slots when passed by value */
struct bpf_testmod_arena_pair {
u64 a;
u64 b;
};
struct bpf_testmod_ops3 {
int (*test_1)(void);
int (*test_2)(void);
@ -112,6 +118,8 @@ struct bpf_testmod_ops3 {
/* enough leading args to force @ptr onto the stack on x86 and arm64 */
int (*test_arena_stack)(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f,
u64 g, u64 h, u64 *ptr);
/* a multi-slot leading arg, so @ptr is not at the slot its arg index suggests */
int (*test_arena_multislot)(struct bpf_testmod_arena_pair p, u64 *ptr);
};
struct st_ops_args {

1
tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h
View File

@ -123,6 +123,7 @@ void bpf_testmod_test_mod_kfunc(int i) __ksym;
int bpf_testmod_ops3_call_test_arena(__u64 *ptr__arena) __ksym;
int bpf_testmod_ops3_call_test_arena_nullable(__u64 *ptr__arena__nullable) __ksym;
int bpf_testmod_ops3_call_test_arena_stack(__u64 *ptr__arena) __ksym;
int bpf_testmod_ops3_call_test_arena_multislot(__u64 *ptr__arena) __ksym;
__u64 bpf_kfunc_call_test1(struct sock *sk, __u32 a, __u64 b,
__u32 c, __u64 d) __ksym;