github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-05 04:56:13 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'support-bpf_fastcall-patterns-for-calls-to-kfuncs'

Browse Source 
Eduard Zingerman says:

====================
support bpf_fastcall patterns for calls to kfuncs

As an extension of [1], allow bpf_fastcall patterns for kfuncs:
- pattern rules are the same as for helpers;
- spill/fill removal is allowed only for kfuncs listed in the
  is_fastcall_kfunc_call (under assumption that such kfuncs would
  always be members of special_kfunc_list).

Allow bpf_fastcall rewrite for bpf_cast_to_kern_ctx() and
bpf_rdonly_cast() in order to conjure selftests for this feature.

After this patch-set verifier would rewrite the program below:

  r2 = 1
  *(u64 *)(r10 - 32) = r2
  call %[bpf_cast_to_kern_ctx]
  r2 = *(u64 *)(r10 - 32)
  r0 = r2;"

As follows:

  r2 = 1   /* spill/fill at r10[-32] is removed */
  r0 = r1  /* replacement for bpf_cast_to_kern_ctx() */
  r0 = r2
  exit

Also, attribute used by LLVM implementation of the feature had been
changed from no_caller_saved_registers to bpf_fastcall (see [2]).
This patch-set replaces references to nocsr by references to
bpf_fastcall to keep LLVM and Kernel parts in sync.

[1] no_caller_saved_registers attribute for helper calls
    https://lore.kernel.org/bpf/20240722233844.1406874-1-eddyz87@gmail.com/
[2] [BPF] introduce __attribute__((bpf_fastcall))
    https://github.com/llvm/llvm-project/pull/105417

Changes v2->v3:
- added a patch fixing arch_mask handling in test_loader,
  otherwise newly added tests for the feature were skipped
  (a fix for regression introduced by a recent commit);
- fixed warning regarding unused 'params' variable;
- applied stylistical fixes suggested by Yonghong;
- added acks from Yonghong;

Changes v1->v2:
- added two patches replacing all mentions of nocsr by bpf_fastcall
  (suggested by Andrii);
- removed KF_NOCSR flag (suggested by Yonghong).

v1: https://lore.kernel.org/bpf/20240812234356.2089263-1-eddyz87@gmail.com/
v2: https://lore.kernel.org/bpf/20240817015140.1039351-1-eddyz87@gmail.com/
====================

Link: https://lore.kernel.org/r/20240822084112.3257995-1-eddyz87@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov 2024-08-22 08:35:21 -07:00
commit d352eca266
7 changed files with 192 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/linux/bpf.h
View File

@ -808,12 +808,12 @@ struct bpf_func_proto {
bool gpl_only;
bool pkt_access;
bool might_sleep;
/* set to true if helper follows contract for gcc/llvm
* attribute no_caller_saved_registers:
/* set to true if helper follows contract for llvm
* attribute bpf_fastcall:
* - void functions do not scratch r0
* - functions taking N arguments scratch only registers r1-rN
*/
bool allow_nocsr;
bool allow_fastcall;
enum bpf_return_type ret_type;
union {
struct {

18
include/linux/bpf_verifier.h
View File

@ -577,13 +577,13 @@ struct bpf_insn_aux_data {
bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */
u8 alu_state; /* used in combination with alu_limit */
/* true if STX or LDX instruction is a part of a spill/fill
* pattern for a no_caller_saved_registers call.
* pattern for a bpf_fastcall call.
*/
u8 nocsr_pattern:1;
u8 fastcall_pattern:1;
/* for CALL instructions, a number of spill/fill pairs in the
* no_caller_saved_registers pattern.
* bpf_fastcall pattern.
*/
u8 nocsr_spills_num:3;
u8 fastcall_spills_num:3;
/* below fields are initialized once */
unsigned int orig_idx; /* original instruction index */
@ -653,10 +653,10 @@ struct bpf_subprog_info {
u32 linfo_idx; /* The idx to the main_prog->aux->linfo */
u16 stack_depth; /* max. stack depth used by this function */
u16 stack_extra;
/* offsets in range [stack_depth .. nocsr_stack_off)
* are used for no_caller_saved_registers spills and fills.
/* offsets in range [stack_depth .. fastcall_stack_off)
* are used for bpf_fastcall spills and fills.
*/
s16 nocsr_stack_off;
s16 fastcall_stack_off;
bool has_tail_call: 1;
bool tail_call_reachable: 1;
bool has_ld_abs: 1;
@ -664,8 +664,8 @@ struct bpf_subprog_info {
bool is_async_cb: 1;
bool is_exception_cb: 1;
bool args_cached: 1;
/* true if nocsr stack region is used by functions that can't be inlined */
bool keep_nocsr_stack: 1;
/* true if bpf_fastcall stack region is used by functions that can't be inlined */
bool keep_fastcall_stack: 1;
u8 arg_cnt;
struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];

2
kernel/bpf/helpers.c
View File

@ -158,7 +158,7 @@ const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
.func = bpf_get_smp_processor_id,
.gpl_only = false,
.ret_type = RET_INTEGER,
.allow_nocsr = true,
.allow_fastcall = true,
};
BPF_CALL_0(bpf_get_numa_node_id)

181
kernel/bpf/verifier.c
View File

@ -4579,28 +4579,28 @@ static int get_reg_width(struct bpf_reg_state *reg)
return fls64(reg->umax_value);
}
/* See comment for mark_nocsr_pattern_for_call() */
static void check_nocsr_stack_contract(struct bpf_verifier_env *env, struct bpf_func_state *state,
int insn_idx, int off)
/* See comment for mark_fastcall_pattern_for_call() */
static void check_fastcall_stack_contract(struct bpf_verifier_env *env,
struct bpf_func_state *state, int insn_idx, int off)
{
struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
struct bpf_insn_aux_data *aux = env->insn_aux_data;
int i;
if (subprog->nocsr_stack_off <= off || aux[insn_idx].nocsr_pattern)
if (subprog->fastcall_stack_off <= off || aux[insn_idx].fastcall_pattern)
return;
/* access to the region [max_stack_depth .. nocsr_stack_off)
* from something that is not a part of the nocsr pattern,
* disable nocsr rewrites for current subprogram by setting
* nocsr_stack_off to a value smaller than any possible offset.
/* access to the region [max_stack_depth .. fastcall_stack_off)
* from something that is not a part of the fastcall pattern,
* disable fastcall rewrites for current subprogram by setting
* fastcall_stack_off to a value smaller than any possible offset.
*/
subprog->nocsr_stack_off = S16_MIN;
/* reset nocsr aux flags within subprogram,
subprog->fastcall_stack_off = S16_MIN;
/* reset fastcall aux flags within subprogram,
* happens at most once per subprogram
*/
for (i = subprog->start; i < (subprog + 1)->start; ++i) {
aux[i].nocsr_spills_num = 0;
aux[i].nocsr_pattern = 0;
aux[i].fastcall_spills_num = 0;
aux[i].fastcall_pattern = 0;
}
}
@ -4652,7 +4652,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
if (err)
return err;
check_nocsr_stack_contract(env, state, insn_idx, off);
check_fastcall_stack_contract(env, state, insn_idx, off);
mark_stack_slot_scratched(env, spi);
if (reg && !(off % BPF_REG_SIZE) && reg->type == SCALAR_VALUE && env->bpf_capable) {
bool reg_value_fits;
@ -4787,7 +4787,7 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
return err;
}
check_nocsr_stack_contract(env, state, insn_idx, min_off);
check_fastcall_stack_contract(env, state, insn_idx, min_off);
/* Variable offset writes destroy any spilled pointers in range. */
for (i = min_off; i < max_off; i++) {
u8 new_type, *stype;
@ -4926,7 +4926,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
reg = &reg_state->stack[spi].spilled_ptr;
mark_stack_slot_scratched(env, spi);
check_nocsr_stack_contract(env, state, env->insn_idx, off);
check_fastcall_stack_contract(env, state, env->insn_idx, off);
if (is_spilled_reg(&reg_state->stack[spi])) {
u8 spill_size = 1;
@ -5087,7 +5087,7 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env,
min_off = reg->smin_value + off;
max_off = reg->smax_value + off;
mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
check_nocsr_stack_contract(env, ptr_state, env->insn_idx, min_off);
check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
return 0;
}
@ -6804,13 +6804,13 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
struct bpf_insn_aux_data *aux = &env->insn_aux_data[env->insn_idx];
int min_valid_off, max_bpf_stack;
/* If accessing instruction is a spill/fill from nocsr pattern,
/* If accessing instruction is a spill/fill from bpf_fastcall pattern,
* add room for all caller saved registers below MAX_BPF_STACK.
* In case if nocsr rewrite won't happen maximal stack depth
* In case if bpf_fastcall rewrite won't happen maximal stack depth
* would be checked by check_max_stack_depth_subprog().
*/
max_bpf_stack = MAX_BPF_STACK;
if (aux->nocsr_pattern)
if (aux->fastcall_pattern)
max_bpf_stack += CALLER_SAVED_REGS * BPF_REG_SIZE;
if (t == BPF_WRITE || env->allow_uninit_stack)
@ -16118,14 +16118,14 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns,
/* Return a bitmask specifying which caller saved registers are
* clobbered by a call to a helper *as if* this helper follows
* no_caller_saved_registers contract:
* bpf_fastcall contract:
* - includes R0 if function is non-void;
* - includes R1-R5 if corresponding parameter has is described
* in the function prototype.
*/
static u32 helper_nocsr_clobber_mask(const struct bpf_func_proto *fn)
static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)
{
u8 mask;
u32 mask;
int i;
mask = 0;
@ -16138,8 +16138,8 @@ static u32 helper_nocsr_clobber_mask(const struct bpf_func_proto *fn)
}
/* True if do_misc_fixups() replaces calls to helper number 'imm',
* replacement patch is presumed to follow no_caller_saved_registers contract
* (see mark_nocsr_pattern_for_call() below).
* replacement patch is presumed to follow bpf_fastcall contract
* (see mark_fastcall_pattern_for_call() below).
*/
static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
{
@ -16153,7 +16153,30 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
}
}
/* GCC and LLVM define a no_caller_saved_registers function attribute.
/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */
static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
{
u32 vlen, i, mask;
vlen = btf_type_vlen(meta->func_proto);
mask = 0;
if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))
mask |= BIT(BPF_REG_0);
for (i = 0; i < vlen; ++i)
mask |= BIT(BPF_REG_1 + i);
return mask;
}
/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
{
if (meta->btf == btf_vmlinux)
return meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast];
return false;
}
/* LLVM define a bpf_fastcall function attribute.
* This attribute means that function scratches only some of
* the caller saved registers defined by ABI.
* For BPF the set of such registers could be defined as follows:
@ -16163,13 +16186,12 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
*
* The contract between kernel and clang allows to simultaneously use
* such functions and maintain backwards compatibility with old
* kernels that don't understand no_caller_saved_registers calls
* (nocsr for short):
* kernels that don't understand bpf_fastcall calls:
*
* - for nocsr calls clang allocates registers as-if relevant r0-r5
* - for bpf_fastcall calls clang allocates registers as-if relevant r0-r5
* registers are not scratched by the call;
*
* - as a post-processing step, clang visits each nocsr call and adds
* - as a post-processing step, clang visits each bpf_fastcall call and adds
* spill/fill for every live r0-r5;
*
* - stack offsets used for the spill/fill are allocated as lowest
@ -16177,11 +16199,11 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
* purposes;
*
* - when kernel loads a program, it looks for such patterns
* (nocsr function surrounded by spills/fills) and checks if
* spill/fill stack offsets are used exclusively in nocsr patterns;
* (bpf_fastcall function surrounded by spills/fills) and checks if
* spill/fill stack offsets are used exclusively in fastcall patterns;
*
* - if so, and if verifier or current JIT inlines the call to the
* nocsr function (e.g. a helper call), kernel removes unnecessary
* bpf_fastcall function (e.g. a helper call), kernel removes unnecessary
* spill/fill pairs;
*
* - when old kernel loads a program, presence of spill/fill pairs
@ -16200,22 +16222,22 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
* r0 += r2;
* exit;
*
* The purpose of mark_nocsr_pattern_for_call is to:
* The purpose of mark_fastcall_pattern_for_call is to:
* - look for such patterns;
* - mark spill and fill instructions in env->insn_aux_data[*].nocsr_pattern;
* - mark set env->insn_aux_data[*].nocsr_spills_num for call instruction;
* - update env->subprog_info[*]->nocsr_stack_off to find an offset
* at which nocsr spill/fill stack slots start;
* - update env->subprog_info[*]->keep_nocsr_stack.
* - mark spill and fill instructions in env->insn_aux_data[*].fastcall_pattern;
* - mark set env->insn_aux_data[*].fastcall_spills_num for call instruction;
* - update env->subprog_info[*]->fastcall_stack_off to find an offset
* at which bpf_fastcall spill/fill stack slots start;
* - update env->subprog_info[*]->keep_fastcall_stack.
*
* The .nocsr_pattern and .nocsr_stack_off are used by
* check_nocsr_stack_contract() to check if every stack access to
* nocsr spill/fill stack slot originates from spill/fill
* instructions, members of nocsr patterns.
* The .fastcall_pattern and .fastcall_stack_off are used by
* check_fastcall_stack_contract() to check if every stack access to
* fastcall spill/fill stack slot originates from spill/fill
* instructions, members of fastcall patterns.
*
* If such condition holds true for a subprogram, nocsr patterns could
* be rewritten by remove_nocsr_spills_fills().
* Otherwise nocsr patterns are not changed in the subprogram
* If such condition holds true for a subprogram, fastcall patterns could
* be rewritten by remove_fastcall_spills_fills().
* Otherwise bpf_fastcall patterns are not changed in the subprogram
* (code, presumably, generated by an older clang version).
*
* For example, it is *not* safe to remove spill/fill below:
@ -16228,9 +16250,9 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
* r0 += r1; exit;
* exit;
*/
static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,
struct bpf_subprog_info *subprog,
int insn_idx, s16 lowest_off)
static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
struct bpf_subprog_info *subprog,
int insn_idx, s16 lowest_off)
{
struct bpf_insn *insns = env->prog->insnsi, *stx, *ldx;
struct bpf_insn *call = &env->prog->insnsi[insn_idx];
@ -16245,12 +16267,25 @@ static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,
if (get_helper_proto(env, call->imm, &fn) < 0)
/* error would be reported later */
return;
clobbered_regs_mask = helper_nocsr_clobber_mask(fn);
can_be_inlined = fn->allow_nocsr &&
clobbered_regs_mask = helper_fastcall_clobber_mask(fn);
can_be_inlined = fn->allow_fastcall &&
(verifier_inlines_helper_call(env, call->imm) ||
bpf_jit_inlines_helper_call(call->imm));
}
if (bpf_pseudo_kfunc_call(call)) {
struct bpf_kfunc_call_arg_meta meta;
int err;
err = fetch_kfunc_meta(env, call, &meta, NULL);
if (err < 0)
/* error would be reported later */
return;
clobbered_regs_mask = kfunc_fastcall_clobber_mask(&meta);
can_be_inlined = is_fastcall_kfunc_call(&meta);
}
if (clobbered_regs_mask == ALL_CALLER_SAVED_REGS)
return;
@ -16289,36 +16324,36 @@ static void mark_nocsr_pattern_for_call(struct bpf_verifier_env *env,
if (stx->off != off || ldx->off != off)
break;
expected_regs_mask &= ~BIT(stx->src_reg);
env->insn_aux_data[insn_idx - i].nocsr_pattern = 1;
env->insn_aux_data[insn_idx + i].nocsr_pattern = 1;
env->insn_aux_data[insn_idx - i].fastcall_pattern = 1;
env->insn_aux_data[insn_idx + i].fastcall_pattern = 1;
}
if (i == 1)
return;
/* Conditionally set 'nocsr_spills_num' to allow forward
/* Conditionally set 'fastcall_spills_num' to allow forward
* compatibility when more helper functions are marked as
* nocsr at compile time than current kernel supports, e.g:
* bpf_fastcall at compile time than current kernel supports, e.g:
*
* 1: *(u64 *)(r10 - 8) = r1
* 2: call A ;; assume A is nocsr for current kernel
* 2: call A ;; assume A is bpf_fastcall for current kernel
* 3: r1 = *(u64 *)(r10 - 8)
* 4: *(u64 *)(r10 - 8) = r1
* 5: call B ;; assume B is not nocsr for current kernel
* 5: call B ;; assume B is not bpf_fastcall for current kernel
* 6: r1 = *(u64 *)(r10 - 8)
*
* There is no need to block nocsr rewrite for such program.
* Set 'nocsr_pattern' for both calls to keep check_nocsr_stack_contract() happy,
* don't set 'nocsr_spills_num' for call B so that remove_nocsr_spills_fills()
* There is no need to block bpf_fastcall rewrite for such program.
* Set 'fastcall_pattern' for both calls to keep check_fastcall_stack_contract() happy,
* don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()
* does not remove spill/fill pair {4,6}.
*/
if (can_be_inlined)
env->insn_aux_data[insn_idx].nocsr_spills_num = i - 1;
env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;
else
subprog->keep_nocsr_stack = 1;
subprog->nocsr_stack_off = min(subprog->nocsr_stack_off, off);
subprog->keep_fastcall_stack = 1;
subprog->fastcall_stack_off = min(subprog->fastcall_stack_off, off);
}
static int mark_nocsr_patterns(struct bpf_verifier_env *env)
static int mark_fastcall_patterns(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprog = env->subprog_info;
struct bpf_insn *insn;
@ -16335,12 +16370,12 @@ static int mark_nocsr_patterns(struct bpf_verifier_env *env)
continue;
lowest_off = min(lowest_off, insn->off);
}
/* use this offset to find nocsr patterns */
/* use this offset to find fastcall patterns */
for (i = subprog->start; i < (subprog + 1)->start; ++i) {
insn = env->prog->insnsi + i;
if (insn->code != (BPF_JMP | BPF_CALL))
continue;
mark_nocsr_pattern_for_call(env, subprog, i, lowest_off);
mark_fastcall_pattern_for_call(env, subprog, i, lowest_off);
}
}
return 0;
@ -21244,10 +21279,10 @@ static int optimize_bpf_loop(struct bpf_verifier_env *env)
return 0;
}
/* Remove unnecessary spill/fill pairs, members of nocsr pattern,
/* Remove unnecessary spill/fill pairs, members of fastcall pattern,
* adjust subprograms stack depth when possible.
*/
static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)
static int remove_fastcall_spills_fills(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprog = env->subprog_info;
struct bpf_insn_aux_data *aux = env->insn_aux_data;
@ -21258,8 +21293,8 @@ static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)
int i, j;
for (i = 0; i < insn_cnt; i++, insn++) {
if (aux[i].nocsr_spills_num > 0) {
spills_num = aux[i].nocsr_spills_num;
if (aux[i].fastcall_spills_num > 0) {
spills_num = aux[i].fastcall_spills_num;
/* NOPs would be removed by opt_remove_nops() */
for (j = 1; j <= spills_num; ++j) {
*(insn - j) = NOP;
@ -21268,8 +21303,8 @@ static int remove_nocsr_spills_fills(struct bpf_verifier_env *env)
modified = true;
}
if ((subprog + 1)->start == i + 1) {
if (modified && !subprog->keep_nocsr_stack)
subprog->stack_depth = -subprog->nocsr_stack_off;
if (modified && !subprog->keep_fastcall_stack)
subprog->stack_depth = -subprog->fastcall_stack_off;
subprog++;
modified = false;
}
@ -22192,7 +22227,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
if (ret < 0)
goto skip_full_check;
ret = mark_nocsr_patterns(env);
ret = mark_fastcall_patterns(env);
if (ret < 0)
goto skip_full_check;
@ -22209,7 +22244,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
* allocate additional slots.
*/
if (ret == 0)
ret = remove_nocsr_spills_fills(env);
ret = remove_fastcall_spills_fills(env);
if (ret == 0)
ret = check_max_stack_depth(env);

4
tools/testing/selftests/bpf/prog_tests/verifier.c
View File

@ -53,7 +53,7 @@
#include "verifier_movsx.skel.h"
#include "verifier_netfilter_ctx.skel.h"
#include "verifier_netfilter_retcode.skel.h"
#include "verifier_nocsr.skel.h"
#include "verifier_bpf_fastcall.skel.h"
#include "verifier_or_jmp32_k.skel.h"
#include "verifier_precision.skel.h"
#include "verifier_prevent_map_lookup.skel.h"
@ -177,7 +177,7 @@ void test_verifier_meta_access(void) { RUN(verifier_meta_access); }
void test_verifier_movsx(void) { RUN(verifier_movsx); }
void test_verifier_netfilter_ctx(void) { RUN(verifier_netfilter_ctx); }
void test_verifier_netfilter_retcode(void) { RUN(verifier_netfilter_retcode); }
void test_verifier_nocsr(void) { RUN(verifier_nocsr); }
void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); }
void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); }
void test_verifier_precision(void) { RUN(verifier_precision); }
void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); }

81
tools/testing/selftests/bpf/progs/verifier_nocsr.c → tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
View File

@ -2,8 +2,11 @@
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_core_read.h>
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
#include <stdbool.h>
#include "bpf_kfuncs.h"
SEC("raw_tp")
__arch_x86_64
@ -39,7 +42,7 @@ __naked void simple(void)
: __clobber_all);
}
/* The logic for detecting and verifying nocsr pattern is the same for
/* The logic for detecting and verifying bpf_fastcall pattern is the same for
* any arch, however x86 differs from arm64 or riscv64 in a way
* bpf_get_smp_processor_id is rewritten:
* - on x86 it is done by verifier
@ -52,7 +55,7 @@ __naked void simple(void)
*
* It is really desirable to check instruction indexes in the xlated
* patterns, so add this canary test to check that function rewrite by
* jit is correctly processed by nocsr logic, keep the rest of the
* jit is correctly processed by bpf_fastcall logic, keep the rest of the
* tests as x86.
*/
SEC("raw_tp")
@ -463,7 +466,7 @@ __naked static void bad_write_in_subprog_aux(void)
{
asm volatile (
"r0 = 1;"
"*(u64 *)(r1 - 0) = r0;" /* invalidates nocsr contract for caller: */
"*(u64 *)(r1 - 0) = r0;" /* invalidates bpf_fastcall contract for caller: */
"exit;" /* caller stack at -8 used outside of the pattern */
::: __clobber_all);
}
@ -480,7 +483,7 @@ __naked void bad_helper_write(void)
{
asm volatile (
"r1 = 1;"
/* nocsr pattern with stack offset -8 */
/* bpf_fastcall pattern with stack offset -8 */
"*(u64 *)(r10 - 8) = r1;"
"call %[bpf_get_smp_processor_id];"
"r1 = *(u64 *)(r10 - 8);"
@ -488,7 +491,7 @@ __naked void bad_helper_write(void)
"r1 += -8;"
"r2 = 1;"
"r3 = 42;"
/* read dst is fp[-8], thus nocsr rewrite not applied */
/* read dst is fp[-8], thus bpf_fastcall rewrite not applied */
"call %[bpf_probe_read_kernel];"
"exit;"
:
@ -598,7 +601,7 @@ __arch_x86_64
__log_level(4) __msg("stack depth 8")
__xlated("2: r0 = &(void __percpu *)(r0)")
__success
__naked void helper_call_does_not_prevent_nocsr(void)
__naked void helper_call_does_not_prevent_bpf_fastcall(void)
{
asm volatile (
"r1 = 1;"
@ -689,7 +692,7 @@ __naked int bpf_loop_interaction1(void)
{
asm volatile (
"r1 = 1;"
/* nocsr stack region at -16, but could be removed */
/* bpf_fastcall stack region at -16, but could be removed */
"*(u64 *)(r10 - 16) = r1;"
"call %[bpf_get_smp_processor_id];"
"r1 = *(u64 *)(r10 - 16);"
@ -729,7 +732,7 @@ __naked int bpf_loop_interaction2(void)
{
asm volatile (
"r1 = 42;"
/* nocsr stack region at -16, cannot be removed */
/* bpf_fastcall stack region at -16, cannot be removed */
"*(u64 *)(r10 - 16) = r1;"
"call %[bpf_get_smp_processor_id];"
"r1 = *(u64 *)(r10 - 16);"
@ -759,8 +762,8 @@ __msg("stack depth 512+0")
__xlated("r0 = &(void __percpu *)(r0)")
__success
/* cumulative_stack_depth() stack usage is MAX_BPF_STACK,
* called subprogram uses an additional slot for nocsr spill/fill,
* since nocsr spill/fill could be removed the program still fits
* called subprogram uses an additional slot for bpf_fastcall spill/fill,
* since bpf_fastcall spill/fill could be removed the program still fits
* in MAX_BPF_STACK and should be accepted.
*/
__naked int cumulative_stack_depth(void)
@ -798,7 +801,7 @@ __xlated("3: r0 = &(void __percpu *)(r0)")
__xlated("4: r0 = *(u32 *)(r0 +0)")
__xlated("5: exit")
__success
__naked int nocsr_max_stack_ok(void)
__naked int bpf_fastcall_max_stack_ok(void)
{
asm volatile(
"r1 = 42;"
@ -820,7 +823,7 @@ __arch_x86_64
__log_level(4)
__msg("stack depth 520")
__failure
__naked int nocsr_max_stack_fail(void)
__naked int bpf_fastcall_max_stack_fail(void)
{
asm volatile(
"r1 = 42;"
@ -828,7 +831,7 @@ __naked int nocsr_max_stack_fail(void)
"*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;"
"call %[bpf_get_smp_processor_id];"
"r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);"
/* call to prandom blocks nocsr rewrite */
/* call to prandom blocks bpf_fastcall rewrite */
"*(u64 *)(r10 - %[max_bpf_stack_8]) = r1;"
"call %[bpf_get_prandom_u32];"
"r1 = *(u64 *)(r10 - %[max_bpf_stack_8]);"
@ -842,4 +845,56 @@ __naked int nocsr_max_stack_fail(void)
);
}
SEC("cgroup/getsockname_unix")
__xlated("0: r2 = 1")
/* bpf_cast_to_kern_ctx is replaced by a single assignment */
__xlated("1: r0 = r1")
__xlated("2: r0 = r2")
__xlated("3: exit")
__success
__naked void kfunc_bpf_cast_to_kern_ctx(void)
{
asm volatile (
"r2 = 1;"
"*(u64 *)(r10 - 32) = r2;"
"call %[bpf_cast_to_kern_ctx];"
"r2 = *(u64 *)(r10 - 32);"
"r0 = r2;"
"exit;"
:
: __imm(bpf_cast_to_kern_ctx)
: __clobber_all);
}
SEC("raw_tp")
__xlated("3: r3 = 1")
/* bpf_rdonly_cast is replaced by a single assignment */
__xlated("4: r0 = r1")
__xlated("5: r0 = r3")
void kfunc_bpf_rdonly_cast(void)
{
asm volatile (
"r2 = %[btf_id];"
"r3 = 1;"
"*(u64 *)(r10 - 32) = r3;"
"call %[bpf_rdonly_cast];"
"r3 = *(u64 *)(r10 - 32);"
"r0 = r3;"
:
: __imm(bpf_rdonly_cast),
[btf_id]"r"(bpf_core_type_id_kernel(union bpf_attr))
: __clobber_common);
}
/* BTF FUNC records are not generated for kfuncs referenced
* from inline assembly. These records are necessary for
* libbpf to link the program. The function below is a hack
* to ensure that BTF FUNC records are generated.
*/
void kfunc_root(void)
{
bpf_cast_to_kern_ctx(0);
bpf_rdonly_cast(0, 0);
}
char _license[] SEC("license") = "GPL";

2
tools/testing/selftests/bpf/test_loader.c
View File

@ -543,7 +543,7 @@ static int parse_test_spec(struct test_loader *tester,
}
}
spec->arch_mask = arch_mask;
spec->arch_mask = arch_mask ?: -1;
if (spec->mode_mask == 0)
spec->mode_mask = PRIV;