github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-09 09:22:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

bpf-fixes

Browse Source 
-----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+soXsSLHKoYyzcli6rmadz2vbToFAmqdhRMACgkQ6rmadz2v
 bTo8vRAAnP+x1z6FBOgHawGmmBpXtazoipRFeok/+/YGQ4SbS+evvFNvmkeCyihZ
 7EeNHGuWEbA6xQBAcMf0StMjmr3jkJvUA6lsHSxBiFLpHGqNt0Fz9IXvAL/8QWPg
 F7+CBs5A+J61i66LEmINbWzW1ujDf7baU/1VWui9zfPxnmPu40V/74eXzm6DmH/V
 oHTsCscCYNe+N+Wix6B6nLrObY537fcJU12N56uqpkaSc0+6H1fBQJmyFGpYroTX
 +4g+DNzEHyIYmt6B3+oDpbT0Zh/pP6ROq5TPDNJSSa+5B/uPo5C5pH520HwXF9hJ
 04uaD7y2fpE809Nwl3OEK4ozpJ+mEGU1NKZPvFAxckY6GCiBtfbGd1etFno7r+4F
 mZhNBKbQVEQX4XJUlcoqn9n7T2OCDnjiuneZAjIi4vUdevnSnakr0rCTFriiHP54
 jg4fEkXxvYjPiqs5SQbxoGxmYSfg9fYaSfK4HgFdlfbwp+3Vmv4BW/tcsIJ9dcPs
 L3YWkUU3znjbu2BLy0CP9fGt/1ik8p4/tA8vvZwz0yqpDMJkLy1TziGMef51UQ7k
 1Pe/Ln5J3C6xnWs+3TRF11y8PuuCF0y47DGPACfwPoQxca1ufJ5fgmLXRRQToZe4
 RjDmoeQvvI9w9kxh6gVhTHoiiEeD0wcQqx2CMHKTrU/PbM7d4Co=
 =oz4Z
 -----END PGP SIGNATURE-----

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Pull bpf fixes from Alexei Starovoitov:
 "This mainly contains verifier fixes that address bugs reported by
  Nicholas Carlini.

   - Fix incorrect non-NULL inference in pointer comparisons: pointer
     types that may be NULL at runtime, pointers with unbounded offsets,
     JMP32 comparisons with zero, and imprecise zero registers (Eduard
     Zingerman)

   - Fix precision tracking for half-dead zero spills, ld_abs/ld_ind
     implicit subprog exit, bpf_loop() callbacks, linked scalar ids and
     NULL call arguments (Eduard Zingerman)

   - Reject BPF_PSEUDO_FUNC reference to the main program, fix zero
     extension of arena 32-bit cmpxchg, don't rewrite bpf_fastcall
     patterns entered by a jump (Eduard Zingerman)

   - Fix percpu map update and BPF_F_CPU validation with sparse CPU IDs
     (Hui Su)

   - Fix NULL-ptr-derefs in bpf_snprintf_btf() for void and VAR types,
     and reject key-less BTF for hash maps (Jiayuan Chen)

   - Various fixes (Kumar Kartikeya Dwivedi):
       - Fix out-of-bounds access in disassembler on invalid LDSX
         instruction
       - mark siginfo of signal tracepoints as scalar and
         sched_process_wait argument as nullable
       - mark faultable stack helpers as sleepable
       - reject tail calls and legacy packet loads from callbacks
       - enforce rbtree callback lock restrictions for resilient locks
       - require MEM_PERCPU for percpu kptr stores
       - clear NON_OWN_REF after RCU protection ends
       - mark NULL kptr stores precise
       - preserve inner map identity in callback frames
       - reject non-scalar bpf_loop() iteration counts

   - Fix trampoline allocation slowdown on x86 by using
     EXECMEM_MODULE_DATA (Mike Rapoport)

   - Keep bpf_refcount_acquire() nullable for borrowed RCU kptrs and
     reject untrusted allocated-object pointers (Ning Ding)

   - Fix special fields handling in recycled rhtab elements (Nuoqi Gui,
     Yuan Chen)"

* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (86 commits)
  bpf, riscv: Make arena support depend on ZACAS
  selftests/bpf: Test pointer bpf_loop iteration count rejection
  bpf: Reject non-scalar bpf_loop iteration counts
  bpf: use mark_arg_precision() in check_mem_size_reg()
  bpf: propagate mark_chain_precision() errors out of loop_flag_is_zero()
  selftests/bpf: precision of a NULL global subprogram BTF_ID argument
  bpf: mark a NULL BTF_ID argument of a global subprogram precise
  selftests/bpf: precision of a NULL kfunc argument
  bpf: mark a NULL kfunc argument precise
  selftests/bpf: precision of a NULL global subprogram memory argument
  bpf: mark a NULL memory argument of a call precise
  selftests/bpf: precision of a NULL helper argument
  bpf: mark a NULL call argument precise
  selftests/bpf: Test inner map identities in callbacks
  bpf: Preserve inner map identity in callback frames
  selftests/bpf: Test imprecise scalar kptr stores
  bpf: Mark NULL kptr stores precise
  selftests/bpf: Test rhtab kptr cancellation semantics
  bpf: Cancel special fields when recycling rhtab elements
  selftests/bpf: Test timer field on recycled rhtab element
  ...
This commit is contained in:
Linus Torvalds 2026-09-06 13:49:44 -07:00
commit 2beb1b31a1
58 changed files with 2556 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
arch/riscv/net/bpf_jit_comp64.c
View File

@ -2128,7 +2128,15 @@ bool bpf_jit_supports_ptr_xchg(void)
bool bpf_jit_supports_arena(void)
{
return true;
/*
* The arena range tree uses kmalloc_nolock(), which needs
* cmpxchg128, provided by ZACAS on riscv.
*/
#ifdef system_has_cmpxchg128
return system_has_cmpxchg128();
#else
return false;
#endif
}
bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)

8
arch/x86/net/bpf_jit_comp.c
View File

@ -13,6 +13,7 @@
#include <linux/bpf_verifier.h>
#include <linux/memory.h>
#include <linux/sort.h>
#include <linux/execmem.h>
#include <asm/extable.h>
#include <asm/ftrace.h>
#include <asm/set_memory.h>
@ -3818,15 +3819,16 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
*
* We cannot use kvmalloc here, because we need image to be in
* module memory range.
* Since it must be writable use bpf_jit_alloc_exec_rw().
* Since it must be writable use execmem_alloc(EXECMEM_MODULE_DATA)
* that returns writable memory in the module address space.
*/
image = bpf_jit_alloc_exec_rw(PAGE_SIZE);
image = execmem_alloc(EXECMEM_MODULE_DATA, PAGE_SIZE);
if (!image)
return -ENOMEM;
ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image,
m, flags, tnodes, func_addr);
bpf_jit_free_exec(image);
execmem_free(image);
return ret;
}

3
include/linux/bpf.h
View File

@ -894,6 +894,7 @@ enum bpf_arg_type {
ARG_PTR_TO_CTX, /* pointer to context */
ARG_ANYTHING, /* any (initialized) argument is ok */
ARG_SCALAR, /* scalar argument */
ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */
ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
@ -4209,7 +4210,7 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all
return -EINVAL;
cpu = flags >> 32;
if ((flags & BPF_F_CPU) && cpu >= num_possible_cpus())
if ((flags & BPF_F_CPU) && (cpu >= nr_cpu_ids || !cpu_possible(cpu)))
return -ERANGE;
}

16
include/linux/bpf_verifier.h
View File

@ -706,6 +706,8 @@ struct bpf_insn_aux_data {
*/
u32 calls_callback:1;
u32 indirect_target:1; /* if it is an indirect jump target */
/* true if some jump or call instruction targets this instruction */
u32 jump_target:1;
/*
* CFG strongly connected component this instruction belongs to,
* zero if it is a singleton SCC.
@ -1142,6 +1144,16 @@ static inline void mark_jmp_point(struct bpf_verifier_env *env, int idx)
env->insn_aux_data[idx].jmp_point = true;
}
static inline void mark_jump_target(struct bpf_verifier_env *env, int idx)
{
env->insn_aux_data[idx].jump_target = true;
}
static inline bool bpf_is_jump_target(struct bpf_verifier_env *env, int insn_idx)
{
return env->insn_aux_data[insn_idx].jump_target;
}
static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
{
struct bpf_verifier_state *cur = env->cur_state;
@ -1369,7 +1381,9 @@ static inline bool bpf_type_has_unsafe_modifiers(u32 type)
static inline bool type_is_ptr_alloc_obj(u32 type)
{
return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC;
return base_type(type) == PTR_TO_BTF_ID &&
type_flag(type) & MEM_ALLOC &&
!(type_flag(type) & PTR_UNTRUSTED);
}
static inline bool type_is_non_owning_ref(u32 type)

1
include/linux/filter.h
View File

@ -1376,7 +1376,6 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
void bpf_jit_binary_free(struct bpf_binary_header *hdr);
u64 bpf_jit_alloc_exec_limit(void);
void *bpf_jit_alloc_exec(unsigned long size);
void *bpf_jit_alloc_exec_rw(unsigned long size);
void bpf_jit_free_exec(void *addr);
void bpf_jit_free(struct bpf_prog *fp);
struct bpf_binary_header *

5
kernel/bpf/arraymap.c
View File

@ -436,7 +436,7 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
void __percpu *pptr;
void *ptr, *val;
u32 size;
int cpu;
int cpu, off = 0;
if (unlikely((map_flags & BPF_F_LOCK) || (u32)map_flags > BPF_F_ALL_CPUS))
/* unknown flags */
@ -468,9 +468,10 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
}
for_each_possible_cpu(cpu) {
ptr = per_cpu_ptr(pptr, cpu);
val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off;
copy_map_value(map, ptr, val);
bpf_obj_cancel_fields(map, ptr);
off += size;
}
unlock:
rcu_read_unlock();

64
kernel/bpf/backtrack.c
View File

@ -520,24 +520,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
return -EFAULT;
}
} else if (opcode == BPF_EXIT) {
bool r0_precise;
/* Backtracking to a nested function call, 'idx' is a part of
* the inner frame 'subseq_idx' is a part of the outer frame.
* In case of a regular function call, instructions giving
* precision to registers R1-R5 should have been found already.
* In case of a callback, it is ok to have R1-R5 marked for
* backtracking, as these registers are set by the function
* invoking callback.
*/
if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx))
for (i = BPF_REG_1; i <= BPF_REG_5; i++)
bt_clear_reg(bt, i);
if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
verifier_bug(env, "backtracking exit unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
bool from_subprog_call, r0_precise;
/* BPF_EXIT in subprog or callback always returns
* right after the call instruction, so by checking
@ -547,9 +530,23 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
* case, we need to propagate r0 precision, if
* necessary. In the former we never do that.
*/
r0_precise = subseq_idx - 1 >= 0 &&
bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) &&
bt_is_reg_set(bt, BPF_REG_0);
from_subprog_call = subseq_idx - 1 >= 0 &&
bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]);
r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0);
/* Backtracking to a nested function call, 'idx' is a part of
* the inner frame 'subseq_idx' is a part of the outer frame.
* In case of a regular function call, instructions giving
* precision to registers R1-R5 should have been found already.
* In case of a callback from bpf_loop(), R{1,4} in the calling
* frame would be set as precise and that is correct.
*/
if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) {
verifier_bug(env, "backtracking exit unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
bt_clear_reg(bt, BPF_REG_0);
if (bt_subprog_enter(bt))
@ -582,16 +579,29 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
*/
}
} else if (class == BPF_LD) {
if (!bt_is_reg_set(bt, dreg))
return 0;
bt_clear_reg(bt, dreg);
/* It's ld_imm64 or ld_abs or ld_ind.
* For ld_imm64 no further tracking of precision
* into parent is necessary
*/
if (mode == BPF_IND || mode == BPF_ABS)
/* to be analyzed */
return -ENOTSUPP;
if (mode == BPF_IMM) {
bt_clear_reg(bt, dreg);
return 0;
}
/*
* BPF_{IND,ABS} are modelled as two branches:
* - fallthrough;
* - implicit subprogram exit.
* It is necessary to switch current frame if
* implicit subprogram exit branch is backtracked.
*/
if (mode == BPF_IND || mode == BPF_ABS) {
if (bt_is_reg_set(bt, dreg))
return -ENOTSUPP;
if (subseq_idx != idx + 1)
if (bt_subprog_enter(bt))
return -EFAULT;
return 0;
}
}
/* Propagate precision marks to linked registers, to account for
* registers marked as precise in this function.

2
kernel/bpf/bpf_iter.c
View File

@ -754,7 +754,7 @@ const struct bpf_func_proto bpf_loop_proto = {
.func = bpf_loop,
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_ANYTHING,
.arg1_type = ARG_SCALAR,
.arg2_type = ARG_PTR_TO_FUNC,
.arg3_type = ARG_PTR_TO_STACK_OR_NULL,
.arg4_type = ARG_ANYTHING,

27
kernel/bpf/btf.c
View File

@ -2911,14 +2911,29 @@ static void btf_modifier_show(const struct btf *btf,
else
t = btf_type_skip_modifiers(btf, type_id, NULL);
btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
/*
* A modifier can resolve to void, which has no show op; print a
* placeholder rather than dereferencing NULL.
*/
if (!btf_type_ops(t))
btf_df_show(btf, t, type_id, data, bits_offset, show);
else
btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
}
static void btf_var_show(const struct btf *btf, const struct btf_type *t,
u32 type_id, void *data, u8 bits_offset,
struct btf_show *show)
{
t = btf_type_id_resolve(btf, &type_id);
/*
* btf_type_id_resolve() dereferences btf->resolved_ids, which is NULL
* for a base BTF (e.g. the vmlinux BTF that bpf_snprintf_btf() uses).
* Resolve the var's type directly in that case.
*/
if (btf->resolved_ids)
t = btf_type_id_resolve(btf, &type_id);
else
t = btf_type_skip_modifiers(btf, t->type, &type_id);
btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
}
@ -6657,6 +6672,10 @@ struct bpf_raw_tp_null_args {
static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
/* sched */
{ "sched_pi_setprio", 0x10 },
/*
* do_wait() passes NULL for wait4(-1) and waitid(P_ALL).
*/
{ "sched_process_wait", 0x1 },
/* ... from sched_numa_pair_template event class */
{ "sched_stick_numa", 0x100 },
{ "sched_swap_numa", 0x100 },
@ -6717,6 +6736,9 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = {
{ "rxrpc_resend", 0x10 },
{ "rxrpc_tq", 0x10 },
{ "rxrpc_client", 0x1 },
/* signal */
{ "signal_generate", 0x20 },
{ "signal_deliver", 0x20 },
/* skb */
{"kfree_skb", 0x1000},
/* sunrpc */
@ -8727,6 +8749,7 @@ BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int
const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = {
.func = bpf_btf_find_by_name_kind,
.gpl_only = false,
.might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY,
.arg2_type = ARG_MEM_SIZE,

3
kernel/bpf/cfg.c
View File

@ -125,6 +125,7 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
/* mark branch target for state pruning */
mark_prune_point(env, w);
mark_jmp_point(env, w);
mark_jump_target(env, w);
}
if (insn_state[w] == 0) {
@ -403,6 +404,7 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env)
}
mark_jmp_point(env, w);
mark_jump_target(env, w);
/* EXPLORED || DISCOVERED */
if (insn_state[w])
@ -564,6 +566,7 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
mark_prune_point(env, t + off + 1);
mark_jmp_point(env, t + off + 1);
mark_jump_target(env, t + off + 1);
return ret;

5
kernel/bpf/core.c
View File

@ -1128,11 +1128,6 @@ void *bpf_jit_alloc_exec(unsigned long size)
return execmem_alloc(EXECMEM_BPF, size);
}
void *bpf_jit_alloc_exec_rw(unsigned long size)
{
return execmem_alloc_rw(EXECMEM_BPF, size);
}
void bpf_jit_free_exec(void *addr)
{
execmem_free(addr);

50
kernel/bpf/disasm.c
View File

@ -7,6 +7,9 @@
#include "disasm.h"
/* Only defined by the non-UAPI linux/filter.h, which this file cannot use. */
#define BPF_PROBE_ATOMIC 0xe0
#define __BPF_FUNC_STR_FN(x) [BPF_FUNC_ ## x] = __stringify(bpf_ ## x)
static const char * const func_id_str[] = {
__BPF_FUNC_MAPPER(__BPF_FUNC_STR_FN)
@ -226,57 +229,57 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
insn->imm);
}
} else if (class == BPF_STX) {
const char *probe_pfx = BPF_MODE(insn->code) == BPF_PROBE_ATOMIC ? "probe " : "";
bool atomic = BPF_MODE(insn->code) == BPF_ATOMIC ||
BPF_MODE(insn->code) == BPF_PROBE_ATOMIC;
if (BPF_MODE(insn->code) == BPF_MEM)
verbose(cbs->private_data, "(%02x) *(%s *)(r%d %+d) = r%d",
insn->code,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg,
insn->off, insn->src_reg);
else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
else if (atomic &&
(insn->imm == BPF_ADD || insn->imm == BPF_AND ||
insn->imm == BPF_OR || insn->imm == BPF_XOR)) {
verbose(cbs->private_data, "(%02x) lock *(%s *)(r%d %+d) %s r%d",
insn->code,
verbose(cbs->private_data, "(%02x) %slock *(%s *)(r%d %+d) %s r%d",
insn->code, probe_pfx,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off,
bpf_alu_string[BPF_OP(insn->imm) >> 4],
insn->src_reg);
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
} else if (atomic &&
(insn->imm == (BPF_ADD | BPF_FETCH) ||
insn->imm == (BPF_AND | BPF_FETCH) ||
insn->imm == (BPF_OR | BPF_FETCH) ||
insn->imm == (BPF_XOR | BPF_FETCH))) {
verbose(cbs->private_data, "(%02x) r%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)",
insn->code, insn->src_reg,
verbose(cbs->private_data, "(%02x) %sr%d = atomic%s_fetch_%s((%s *)(r%d %+d), r%d)",
insn->code, probe_pfx, insn->src_reg,
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
bpf_atomic_alu_string[BPF_OP(insn->imm) >> 4],
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off, insn->src_reg);
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
insn->imm == BPF_CMPXCHG) {
verbose(cbs->private_data, "(%02x) r0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)",
insn->code,
} else if (atomic && insn->imm == BPF_CMPXCHG) {
verbose(cbs->private_data, "(%02x) %sr0 = atomic%s_cmpxchg((%s *)(r%d %+d), r0, r%d)",
insn->code, probe_pfx,
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off,
insn->src_reg);
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
insn->imm == BPF_XCHG) {
verbose(cbs->private_data, "(%02x) r%d = atomic%s_xchg((%s *)(r%d %+d), r%d)",
insn->code, insn->src_reg,
} else if (atomic && insn->imm == BPF_XCHG) {
verbose(cbs->private_data, "(%02x) %sr%d = atomic%s_xchg((%s *)(r%d %+d), r%d)",
insn->code, probe_pfx, insn->src_reg,
BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off, insn->src_reg);
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
insn->imm == BPF_LOAD_ACQ) {
verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))",
insn->code, insn->dst_reg,
} else if (atomic && insn->imm == BPF_LOAD_ACQ) {
verbose(cbs->private_data, "(%02x) %sr%d = load_acquire((%s *)(r%d %+d))",
insn->code, probe_pfx, insn->dst_reg,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->src_reg, insn->off);
} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
insn->imm == BPF_STORE_REL) {
verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)",
insn->code,
} else if (atomic && insn->imm == BPF_STORE_REL) {
verbose(cbs->private_data, "(%02x) %sstore_release((%s *)(r%d %+d), r%d)",
insn->code, probe_pfx,
bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
insn->dst_reg, insn->off, insn->src_reg);
} else {
@ -295,7 +298,8 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs,
verbose(cbs->private_data, "BUG_st_%02x", insn->code);
}
} else if (class == BPF_LDX) {
if (BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) {
if ((BPF_MODE(insn->code) != BPF_MEM && BPF_MODE(insn->code) != BPF_MEMSX) ||
(BPF_MODE(insn->code) == BPF_MEMSX && BPF_SIZE(insn->code) == BPF_DW)) {
verbose(cbs->private_data, "BUG_ldx_%02x", insn->code);
return;
}

7
kernel/bpf/fixups.c
View File

@ -13,10 +13,15 @@
#define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
/*
* Matches BPF_PROBE_ATOMIC too: bpf_convert_ctx_accesses() rewrites arena
* atomics before bpf_opt_subreg_zext_lo32_rnd_hi32() runs.
*/
static bool is_cmpxchg_insn(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_STX &&
BPF_MODE(insn->code) == BPF_ATOMIC &&
(BPF_MODE(insn->code) == BPF_ATOMIC ||
BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) &&
insn->imm == BPF_CMPXCHG;
}

29
kernel/bpf/hashtab.c
View File

@ -530,6 +530,9 @@ static int htab_map_check_btf(struct bpf_map *map, const struct btf *btf,
{
struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
if (btf_type_is_void(key_type))
return -EINVAL;
if (htab_is_prealloc(htab))
return 0;
/*
@ -1025,7 +1028,7 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
} else {
u32 size = round_up(htab->map.value_size, 8);
void *val;
int cpu;
int cpu, off = 0;
if (map_flags & BPF_F_CPU) {
cpu = map_flags >> 32;
@ -1037,9 +1040,10 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
for_each_possible_cpu(cpu) {
ptr = per_cpu_ptr(pptr, cpu);
val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off;
copy_map_value(&htab->map, ptr, val);
bpf_obj_cancel_fields(&htab->map, ptr);
off += size;
}
}
}
@ -2864,16 +2868,6 @@ static int rhtab_map_alloc_check(union bpf_attr *attr)
return htab_map_alloc_check(attr);
}
static void rhtab_check_and_free_fields(struct bpf_rhtab *rhtab,
struct rhtab_elem *elem)
{
if (IS_ERR_OR_NULL(rhtab->map.record))
return;
bpf_obj_free_fields(rhtab->map.record,
rhtab_elem_value(elem, rhtab->map.key_size));
}
static void rhtab_mem_dtor(void *obj, void *ctx)
{
struct htab_btf_record *hrec = ctx;
@ -2963,8 +2957,8 @@ static int rhtab_delete_elem(struct bpf_rhtab *rhtab, struct rhtab_elem *elem, v
rhtab_read_elem_value(&rhtab->map, copy, elem, flags);
check_and_init_map_value(&rhtab->map, copy);
}
/* Release internal structs: kptr, bpf_timer, task_work, wq */
rhtab_check_and_free_fields(rhtab, elem);
bpf_obj_cancel_fields(&rhtab->map,
rhtab_elem_value(elem, rhtab->map.key_size));
bpf_mem_cache_free_rcu(&rhtab->ma, elem);
return 0;
}
@ -3005,7 +2999,6 @@ static int rhtab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, void
static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *elem, void *value,
u64 map_flags)
{
struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
void *old_val = rhtab_elem_value(elem, map->key_size);
if (map_flags & BPF_NOEXIST)
@ -3025,7 +3018,7 @@ static long rhtab_map_update_existing(struct bpf_map *map, struct rhtab_elem *el
* kptrs/etc. still sit in the slot. Cancel them after the copy
* to match arraymap's update semantics.
*/
rhtab_check_and_free_fields(rhtab, elem);
bpf_obj_cancel_fields(map, old_val);
return 0;
}
@ -3066,7 +3059,6 @@ static long rhtab_map_update_elem(struct bpf_map *map, void *key, void *value, u
memcpy(elem->data, key, map->key_size);
copy_map_value(map, rhtab_elem_value(elem, map->key_size), value);
check_and_init_map_value(map, rhtab_elem_value(elem, map->key_size));
/* Prevent deadlock for NMI programs attempting to take bucket lock */
bpf_disable_instrumentation();
@ -3110,6 +3102,9 @@ static int rhtab_map_check_btf(struct bpf_map *map, const struct btf *btf,
{
struct bpf_rhtab *rhtab = container_of(map, struct bpf_rhtab, map);
if (btf_type_is_void(key_type))
return -EINVAL;
return bpf_ma_set_dtor(map, &rhtab->ma, rhtab_mem_dtor);
}

5
kernel/bpf/local_storage.c
View File

@ -220,7 +220,7 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,
struct bpf_cgroup_storage *storage;
void *val;
u32 size;
int cpu;
int cpu, off = 0;
if ((u32)map_flags & ~(BPF_ANY | BPF_EXIST | BPF_F_CPU | BPF_F_ALL_CPUS))
return -EINVAL;
@ -245,8 +245,9 @@ int bpf_percpu_cgroup_storage_update(struct bpf_map *_map, void *key,
}
size = round_up(_map->value_size, 8);
for_each_possible_cpu(cpu) {
val = (map_flags & BPF_F_ALL_CPUS) ? value : value + size * cpu;
val = (map_flags & BPF_F_ALL_CPUS) ? value : value + off;
copy_map_value(_map, per_cpu_ptr(storage->percpu_buf, cpu), val);
off += size;
}
unlock:
rcu_read_unlock();

35
kernel/bpf/percpu_freelist.c
View File

@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s)
raw_res_spin_lock_init(&head->lock);
head->first = NULL;
}
raw_res_spin_lock_init(&s->extralist.lock);
s->extralist.first = NULL;
return 0;
}
@ -46,22 +48,28 @@ void __pcpu_freelist_push(struct pcpu_freelist *s,
struct pcpu_freelist_node *node)
{
struct pcpu_freelist_head *head;
int cpu;
int cpu, this_cpu;
if (___pcpu_freelist_push(this_cpu_ptr(s->freelist), node))
return;
this_cpu = raw_smp_processor_id();
while (true) {
for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
if (cpu == raw_smp_processor_id())
for_each_cpu_wrap(cpu, cpu_possible_mask, this_cpu) {
if (cpu == this_cpu)
continue;
head = per_cpu_ptr(s->freelist, cpu);
if (raw_res_spin_lock(&head->lock))
continue;
pcpu_freelist_push_node(head, node);
raw_res_spin_unlock(&head->lock);
return;
if (___pcpu_freelist_push(head, node))
return;
}
/*
* Push cannot fail. Use the extra list when none of the
* per-CPU freelists can accept the node.
*/
if (___pcpu_freelist_push(&s->extralist, node))
return;
}
}
@ -117,6 +125,17 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
}
raw_res_spin_unlock(&head->lock);
}
/* Per-CPU lists are empty or unavailable, try the extra list. */
head = &s->extralist;
if (!READ_ONCE(head->first))
return NULL;
if (raw_res_spin_lock(&head->lock))
return NULL;
node = head->first;
if (node)
WRITE_ONCE(head->first, node->next);
raw_res_spin_unlock(&head->lock);
return node;
}

1
kernel/bpf/percpu_freelist.h
View File

@ -14,6 +14,7 @@ struct pcpu_freelist_head {
struct pcpu_freelist {
struct pcpu_freelist_head __percpu *freelist;
struct pcpu_freelist_head extralist;
};
struct pcpu_freelist_node {

2
kernel/bpf/stackmap.c
View File

@ -875,6 +875,7 @@ BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, si
const struct bpf_func_proto bpf_get_stack_sleepable_proto = {
.func = bpf_get_stack_sleepable,
.gpl_only = true,
.might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_CTX,
.arg2_type = ARG_PTR_TO_UNINIT_MEM,
@ -928,6 +929,7 @@ BPF_CALL_4(bpf_get_task_stack_sleepable, struct task_struct *, task, void *, buf
const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = {
.func = bpf_get_task_stack_sleepable,
.gpl_only = false,
.might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_PTR_TO_BTF_ID,
.arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],

11
kernel/bpf/states.c
View File

@ -445,22 +445,19 @@ static void __clean_func_state(struct bpf_verifier_env *env,
struct bpf_reg_state *spill = &st->stack[i].spilled_ptr;
if (lo_live && stype == STACK_SPILL) {
u8 val = STACK_MISC;
if (spill->type != SCALAR_VALUE)
continue;
/*
* 8 byte spill of scalar 0 where half slot is dead
* should become STACK_ZERO in lo 4 bytes.
* Can't replace with STACK_ZERO, because
* that requires bpf_mark_chain_precision().
*/
if (bpf_register_is_null(spill))
val = STACK_ZERO;
continue;
for (j = 0; j < 4; j++) {
u8 *t = &st->stack[i].slot_type[j];
if (*t == STACK_SPILL)
*t = val;
*t = STACK_MISC;
}
}
bpf_mark_reg_not_init(env, spill);

2
kernel/bpf/syscall.c
View File

@ -6568,6 +6568,7 @@ EXPORT_SYMBOL_NS(kern_sys_bpf, "BPF_INTERNAL");
static const struct bpf_func_proto bpf_sys_bpf_proto = {
.func = bpf_sys_bpf,
.gpl_only = false,
.might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_ANYTHING,
.arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY,
@ -6593,6 +6594,7 @@ BPF_CALL_1(bpf_sys_close, u32, fd)
static const struct bpf_func_proto bpf_sys_close_proto = {
.func = bpf_sys_close,
.gpl_only = false,
.might_sleep = true,
.ret_type = RET_INTEGER,
.arg1_type = ARG_ANYTHING,
};

259
kernel/bpf/verifier.c
View File

@ -352,9 +352,18 @@ static bool reg_not_null(struct bpf_verifier_env *env, const struct bpf_reg_stat
if (type_may_be_null(type))
return false;
/*
* The types below guarantee a non-NULL base, an unbounded offset can
* still wrap base + offset to zero.
*/
if (reg_smin(reg) <= -BPF_MAX_VAR_OFF || reg_smax(reg) >= BPF_MAX_VAR_OFF)
return false;
type = base_type(type);
return type == PTR_TO_SOCKET ||
type == PTR_TO_TCP_SOCK ||
type == PTR_TO_XDP_SOCK ||
type == PTR_TO_BUF ||
type == PTR_TO_MAP_VALUE ||
type == PTR_TO_MAP_KEY ||
type == PTR_TO_SOCK_COMMON ||
@ -4237,6 +4246,15 @@ static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx)
return mark_chain_precision_batch(env, env->cur_state);
}
static int mark_arg_precision(struct bpf_verifier_env *env, argno_t argno)
{
int regno = reg_from_argno(argno);
if (regno >= 0)
return mark_chain_precision(env, regno);
return mark_stack_arg_precision(env, arg_idx_from_argno(argno));
}
static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller,
int nargs, const char *callee_name, const struct btf *btf,
const struct btf_param *args)
@ -4486,6 +4504,13 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
if (type_flag(reg->type) & ~perm_flags)
goto bad_type;
/*
* A BPF_KPTR_PERCPU field is read back as MEM_PERCPU, so the value
* stored in it must carry the same flag.
*/
if ((kptr_field->type == BPF_KPTR_PERCPU) != !!(reg->type & MEM_PERCPU))
goto bad_type;
/* We need to verify reg->type and reg->btf, before accessing reg->btf */
reg_name = btf_type_name(reg->btf, reg->btf_id);
@ -4692,8 +4717,15 @@ static int check_map_kptr_access(struct bpf_verifier_env *env,
return ret;
} else if (class == BPF_STX) {
val_reg = reg_state(env, value_regno);
if (!bpf_register_is_null(val_reg) &&
map_kptr_match_type(env, kptr_field, val_reg, value_regno))
if (bpf_register_is_null(val_reg)) {
/*
* This store is valid only because the scalar is known to be
* zero. Mark it precise so another scalar cannot be pruned
* against this state.
*/
return mark_chain_precision(env, value_regno);
}
if (map_kptr_match_type(env, kptr_field, val_reg, value_regno))
return -EACCES;
} else if (class == BPF_ST) {
if (insn->imm) {
@ -5300,6 +5332,15 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
if (!priv_stack_supported)
subprog[idx].priv_stack_mode = NO_PRIV_STACK;
process_func:
if (subprog[idx].has_ld_abs) {
for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
if (subprog[tmp].is_cb) {
verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n");
return -EINVAL;
}
}
}
/* protect against potential stack overflow that might happen when
* bpf2bpf calls get combined with tailcalls. Limit the caller's stack
* depth for such case down to 256 so that the worst case scenario
@ -6020,7 +6061,13 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
return -EACCES;
}
if (type_is_alloc(reg->type) && !type_is_non_owning_ref(reg->type) &&
/*
* A fault-prone allocated object may still be read through a
* BPF_PROBE_MEM load after its lifetime protection ends. Writes
* through such pointers were rejected above.
*/
if (type_is_alloc(reg->type) && !bpf_may_fault_on_deref(reg->type) &&
!type_is_non_owning_ref(reg->type) &&
!(reg->type & MEM_RCU) && !reg_is_referenced(env, reg)) {
verifier_bug(env, "allocated object must have a referenced id");
return -EFAULT;
@ -7113,14 +7160,8 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
if (err && failure)
*failure = BPF_MEM_SIZE_FAIL_MEMORY;
if (!err) {
int regno = reg_from_argno(size_argno);
if (regno >= 0)
err = mark_chain_precision(env, regno);
else
err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno));
}
if (!err)
err = mark_arg_precision(env, size_argno);
return err;
@ -7137,7 +7178,7 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg
int size, err = 0;
if (bpf_register_is_null(reg))
return 0;
return mark_arg_precision(env, argno);
if (known_memory)
*known_memory = true;
@ -7398,10 +7439,14 @@ static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state
lock);
return -EINVAL;
}
/*
* Invalidate non-owning refs before RCU demotion clears their
* NON_OWN_REF flag.
*/
invalidate_non_owning_refs(env);
if (!in_rcu_cs(env))
invalidate_rcu_protected_refs(env);
invalidate_non_owning_refs(env);
}
return 0;
}
@ -8166,6 +8211,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
[ARG_MEM_SIZE] = &scalar_types,
[ARG_MEM_SIZE_OR_ZERO] = &scalar_types,
[ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
[ARG_SCALAR] = &scalar_types,
[ARG_CONST_MAP_PTR] = &const_map_ptr_types,
[ARG_PTR_TO_CTX] = &context_types,
[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
@ -8717,11 +8763,15 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
return err;
}
if (bpf_register_is_null(reg) && type_may_be_null(arg_type))
if (bpf_register_is_null(reg) && type_may_be_null(arg_type)) {
/* A NULL register has a SCALAR_VALUE type, so skip
* type checking.
*/
err = mark_chain_precision(env, regno);
if (err)
return err;
goto skip_type_check;
}
/* arg_btf_id and arg_size are in a union. */
if (base_type(arg_type) == ARG_PTR_TO_BTF_ID ||
@ -9501,7 +9551,7 @@ static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env)
bpf_for_each_reg_in_vstate_mask(env->cur_state, state, reg, stack, clear_mask, ({
if (reg->type & MEM_RCU) {
bpf_diag_mod_begin(env, reg, NULL, BPF_DIAG_MOD_WRITE);
reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL);
reg->type &= ~(MEM_RCU | PTR_MAYBE_NULL | NON_OWN_REF);
reg->type |= PTR_UNTRUSTED;
bpf_diag_mod_end(env);
}
@ -9719,8 +9769,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
struct bpf_call_arg_meta meta;
int err;
if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type))
if (bpf_register_is_null(reg) && type_may_be_null(arg->arg_type)) {
err = mark_arg_precision(env, argno);
if (err)
return err;
continue;
}
memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta,
@ -9976,10 +10030,12 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env,
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
callee->regs[BPF_REG_2].map_uid = caller->regs[BPF_REG_1].map_uid;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid;
/* pointer to stack or null */
callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
@ -10057,6 +10113,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env,
int insn_idx)
{
struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
u32 map_uid = caller->regs[BPF_REG_1].map_uid;
/* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
* callback_fn(struct bpf_map *map, void *key, void *value);
@ -10064,14 +10121,17 @@ static int set_timer_callback_state(struct bpf_verifier_env *env,
callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
callee->regs[BPF_REG_1].map_ptr = map_ptr;
callee->regs[BPF_REG_1].map_uid = map_uid;
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
callee->regs[BPF_REG_2].map_ptr = map_ptr;
callee->regs[BPF_REG_2].map_uid = map_uid;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
callee->regs[BPF_REG_3].map_ptr = map_ptr;
callee->regs[BPF_REG_3].map_uid = map_uid;
/* unused */
bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
@ -10171,6 +10231,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,
int insn_idx)
{
struct bpf_map *map_ptr = caller->regs[BPF_REG_3].map_ptr;
u32 map_uid = caller->regs[BPF_REG_3].map_uid;
/*
* callback_fn(struct bpf_map *map, void *key, void *value);
@ -10178,14 +10239,17 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env,
callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
callee->regs[BPF_REG_1].map_ptr = map_ptr;
callee->regs[BPF_REG_1].map_uid = map_uid;
callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
__mark_reg_known_zero(&callee->regs[BPF_REG_2]);
callee->regs[BPF_REG_2].map_ptr = map_ptr;
callee->regs[BPF_REG_2].map_uid = map_uid;
callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
__mark_reg_known_zero(&callee->regs[BPF_REG_3]);
callee->regs[BPF_REG_3].map_ptr = map_ptr;
callee->regs[BPF_REG_3].map_uid = map_uid;
/* unused */
bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
@ -10233,9 +10297,10 @@ static void account_current_path(struct bpf_verifier_env *env)
frame ? state->frame[frame - 1] : NULL);
}
/* Are we currently verifying the callback for a rbtree helper that must
* be called with lock held? If so, no need to complain about unreleased
* lock
/*
* Are we currently verifying the callback for an rbtree kfunc that must
* be called with a lock held, or one of that callback's subprogs? If so,
* no need to complain about an unreleased lock.
*/
static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
{
@ -10243,17 +10308,19 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
struct bpf_insn *insn = env->prog->insnsi;
struct bpf_func_state *callee;
int kfunc_btf_id;
u32 frame;
if (!state->curframe)
return false;
for (frame = state->curframe; frame; frame--) {
callee = state->frame[frame];
if (!callee->in_callback_fn)
continue;
callee = state->frame[state->curframe];
kfunc_btf_id = insn[callee->callsite].imm;
if (is_rbtree_lock_required_kfunc(kfunc_btf_id))
return true;
}
if (!callee->in_callback_fn)
return false;
kfunc_btf_id = insn[callee->callsite].imm;
return is_rbtree_lock_required_kfunc(kfunc_btf_id);
return false;
}
static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg)
@ -10630,33 +10697,45 @@ static struct bpf_insn_aux_data *cur_aux(const struct bpf_verifier_env *env)
return &env->insn_aux_data[env->insn_idx];
}
static bool loop_flag_is_zero(struct bpf_verifier_env *env)
/* Returns 1 if R4 is a known zero, 0 if it is not, a negative errno on error. */
static int loop_flag_is_zero(struct bpf_verifier_env *env)
{
struct bpf_reg_state *reg = reg_state(env, BPF_REG_4);
bool reg_is_null = bpf_register_is_null(reg);
int err;
if (reg_is_null)
mark_chain_precision(env, BPF_REG_4);
if (!bpf_register_is_null(reg))
return 0;
return reg_is_null;
err = mark_chain_precision(env, BPF_REG_4);
if (err)
return err;
return 1;
}
static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
static int update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
{
struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;
int flag_is_zero;
if (!state->initialized) {
flag_is_zero = loop_flag_is_zero(env);
if (flag_is_zero < 0)
return flag_is_zero;
state->initialized = 1;
state->fit_for_inline = loop_flag_is_zero(env);
state->fit_for_inline = flag_is_zero;
state->callback_subprogno = subprogno;
return;
return 0;
}
if (!state->fit_for_inline)
return;
return 0;
state->fit_for_inline = (loop_flag_is_zero(env) &&
flag_is_zero = loop_flag_is_zero(env);
if (flag_is_zero < 0)
return flag_is_zero;
state->fit_for_inline = (flag_is_zero &&
state->callback_subprogno == subprogno);
return 0;
}
/* Returns whether or not the given map can potentially elide
@ -10868,6 +10947,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
verbose(env, "get_local_storage() doesn't support non-zero flags\n");
return -EINVAL;
}
err = mark_chain_precision(env, BPF_REG_2);
if (err)
return err;
break;
case BPF_FUNC_for_each_map_elem:
err = push_callback_call(env, insn, insn_idx, meta.subprogno,
@ -10885,7 +10967,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
err = check_bpf_snprintf_call(env, regs);
break;
case BPF_FUNC_loop:
update_loop_inline_state(env, meta.subprogno);
err = update_loop_inline_state(env, meta.subprogno);
if (err)
return err;
/* Verifier relies on R1 value to determine if bpf_loop() iteration
* is finished, thus mark it precise.
*/
@ -11226,6 +11310,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
if (env->cur_state->curframe) {
struct bpf_verifier_state *branch;
/*
* A taken tail call is modeled as a return from the current
* frame. A callback frame cannot be left that way because
* prepare_func_exit() would apply its return contract to the
* unknown R0 synthesized below. Stack-depth validation rejects
* this construct anyway.
*/
if (cur_func(env)->in_callback_fn) {
verbose(env, "cannot tail call within callback\n");
return -EINVAL;
}
mark_reg_scratched(env, BPF_REG_0);
branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false);
if (IS_ERR(branch))
@ -12651,8 +12746,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
if (reg_is_referenced(env, reg))
update_ref_obj(&meta->ref_obj, reg);
if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type))
if (bpf_register_is_null(reg) && type_may_be_null(kf_arg_type)) {
ret = mark_arg_precision(env, argno);
if (ret)
return ret;
continue;
}
if (is_kfunc_arg_map(btf, &args[i])) {
ref_id = *reg2btf_ids[CONST_PTR_TO_MAP];
@ -13146,7 +13245,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
bpf_diag_reg_type_plain(env, reg->type));
return -EINVAL;
}
if (!type_is_non_owning_ref(reg->type))
if (!type_is_non_owning_ref(reg->type) && reg_is_referenced(env, reg))
meta->arg_owning_ref = true;
rec = reg_btf_record(reg);
@ -13228,6 +13327,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
{
int flags = PROCESS_RES_LOCK;
if (in_rbtree_lock_required_cb(env)) {
verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n");
return -EACCES;
}
if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
verbose(env, "%s doesn't point to map value or allocated object\n",
reg_arg_name(env, argno));
@ -14560,9 +14664,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
return -EINVAL;
}
/* pointer types do not carry 32-bit bounds at the moment. */
__mark_reg32_unbounded(dst_reg);
if (sanitize_needed(opcode)) {
ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
&info, false);
@ -14570,6 +14671,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn
return sanitize_err(env, insn, ret);
}
/*
* Pointer types do not carry 32-bit bounds at the moment. Blank r32
* only after sanitize_ptr_alu() may have snapshotted dst_reg into a
* speculative path: otherwise reg_bounds_sanity_check() might hit some
* constraints violations.
*/
__mark_reg32_unbounded(dst_reg);
switch (opcode) {
case BPF_ADD:
/*
@ -16295,6 +16404,13 @@ static int is_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_state *r
if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) {
u64 val;
/*
* The low 32 bits of a valid pointer may well be zero, hence
* nothing below applies to a 32-bit comparison.
*/
if (is_jmp32)
return -1;
/* arrange that reg2 is a scalar, and reg1 is a pointer */
if (!is_reg_const(reg2, is_jmp32)) {
opcode = flip_opcode(opcode);
@ -16856,6 +16972,16 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
return err;
}
/*
* Collect the linked registers before env->{true,false}_reg{1,2} setup,
* otherwise ids dropped by collect_linked_regs() would be resurrected
* when env->{true,false}_reg{1,2} are copied back.
*/
if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id)
collect_linked_regs(env, this_branch, src_reg->id, &linked_regs);
if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);
is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
env->false_reg1 = *dst_reg;
env->false_reg2 = *src_reg;
@ -16910,10 +17036,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
* 'this_branch' and 'other_branch' share this history
* if parent state is created.
*/
if (BPF_SRC(insn->code) == BPF_X && src_reg->type == SCALAR_VALUE && src_reg->id)
collect_linked_regs(env, this_branch, src_reg->id, &linked_regs);
if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);
if (linked_regs.cnt > 1) {
err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs));
if (err)
@ -16963,7 +17085,6 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
*/
if (!is_jmp32 && BPF_SRC(insn->code) == BPF_X &&
__is_pointer_value(false, src_reg) && __is_pointer_value(false, dst_reg) &&
type_may_be_null(src_reg->type) != type_may_be_null(dst_reg->type) &&
base_type(src_reg->type) != PTR_TO_BTF_ID &&
base_type(dst_reg->type) != PTR_TO_BTF_ID) {
eq_branch_regs = NULL;
@ -16979,9 +17100,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
break;
}
if (eq_branch_regs) {
if (type_may_be_null(src_reg->type))
/* src == dst && dst != NULL => src != NULL */
if (reg_not_null(env, dst_reg) && type_may_be_null(src_reg->type))
mark_ptr_not_null_reg(&eq_branch_regs[insn->src_reg]);
else
/* src == dst && src != NULL => dst != NULL */
if (reg_not_null(env, src_reg) && type_may_be_null(dst_reg->type))
mark_ptr_not_null_reg(&eq_branch_regs[insn->dst_reg]);
}
}
@ -16996,6 +17119,15 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
type_may_be_null(dst_reg->type) &&
((BPF_SRC(insn->code) == BPF_K && insn->imm == 0) ||
(BPF_SRC(insn->code) == BPF_X && bpf_register_is_null(src_reg)))) {
/*
* For BPF_X the zero is a property of this execution path,
* hence src_reg has to be precise.
*/
if (BPF_SRC(insn->code) == BPF_X) {
err = mark_chain_precision(env, insn->src_reg);
if (err)
return err;
}
/* Mark all identical registers in each branch as either
* safe or unknown depending R == 0 or R != 0 conditional.
*/
@ -17081,6 +17213,15 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
verbose(env, "callback function not static\n");
return -EINVAL;
}
/*
* When env->subprog_cnt == 1 this instruction won't be rewritten
* to hold a real function address. Assume that no usable program
* combines e.g. main and timer callback and just reject here.
*/
if (subprogno == 0) {
verbose(env, "callback function cannot be the main program\n");
return -EINVAL;
}
dst_reg->type = PTR_TO_FUNC;
dst_reg->subprogno = subprogno;
@ -17146,6 +17287,7 @@ static bool may_access_skb(enum bpf_prog_type type)
*/
static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
{
struct bpf_verifier_state *state = env->cur_state;
struct bpf_reg_state *regs = cur_regs(env);
static const int ctx_reg = BPF_REG_6;
u8 mode = BPF_MODE(insn->code);
@ -17156,6 +17298,13 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
return -EINVAL;
}
for (i = state->curframe; i; i--) {
if (state->frame[i]->in_callback_fn) {
verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n");
return -EINVAL;
}
}
if (!env->ops->gen_ld_abs) {
verifier_bug(env, "gen_ld_abs is null");
return -EFAULT;
@ -17623,6 +17772,10 @@ bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
* r0 = *(u64 *)(r10 - 8); r0 += r1;
* r0 += r1; exit;
* exit;
*
* Both uses of the marks assume that a pattern is entered at its first
* spill and thus executes as a unit, hence a pattern is not grown past
* an instruction targeted by a jump.
*/
static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
struct bpf_subprog_info *subprog,
@ -17661,6 +17814,10 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env,
for (i = 1, off = lowest_off; i <= ARRAY_SIZE(caller_saved); ++i, off += BPF_REG_SIZE) {
if (insn_idx - i < 0 || insn_idx + i >= env->prog->len)
break;
/* stx/ldx/call must not be a jump targets, a jump to the first stx is fine */
if (bpf_is_jump_target(env, insn_idx - i + 1) ||
bpf_is_jump_target(env, insn_idx + i))
break;
stx = &insns[insn_idx - i];
ldx = &insns[insn_idx + i];
/* must be a stack spill/fill pair */

59
tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c Normal file
View File

@ -0,0 +1,59 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include <bpf/btf.h>
/*
* A hash map with a key-less BTF (btf_key_type_id == 0) used to be accepted
* and then NULL-deref in btf_type_show() when dumped through bpffs. A fixed
* kernel rejects it at creation; verify that rejection, with a keyed positive
* control so the -EINVAL is about the missing key type and not some unrelated
* failure.
*/
static void check_keyless(int map_type, __u32 map_flags, int btf_fd, int val_id)
{
LIBBPF_OPTS(bpf_map_create_opts, opts);
int map_fd;
opts.map_flags = map_flags;
opts.btf_fd = btf_fd;
opts.btf_value_type_id = val_id;
/* Positive control: the same map with a real key type is accepted. */
opts.btf_key_type_id = val_id;
map_fd = bpf_map_create(map_type, "keyed_map", 4, 4, 8, &opts);
if (!ASSERT_GE(map_fd, 0, "keyed create is accepted"))
return;
close(map_fd);
/* A key-less BTF must be rejected. */
opts.btf_key_type_id = 0;
map_fd = bpf_map_create(map_type, "keyless_map", 4, 4, 8, &opts);
ASSERT_EQ(map_fd, -EINVAL, "key-less create is rejected");
if (map_fd >= 0)
close(map_fd);
}
void test_btf_map_keyless(void)
{
int btf_fd, val_id;
struct btf *btf;
btf = btf__new_empty();
if (!ASSERT_OK_PTR(btf, "btf__new_empty"))
return;
val_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED);
if (!ASSERT_GT(val_id, 0, "btf__add_int"))
goto out;
if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel"))
goto out;
btf_fd = btf__fd(btf);
if (test__start_subtest("hash"))
check_keyless(BPF_MAP_TYPE_HASH, 0, btf_fd, val_id);
if (test__start_subtest("rhash"))
check_keyless(BPF_MAP_TYPE_RHASH, BPF_F_NO_PREALLOC, btf_fd, val_id);
out:
btf__free(btf);
}

6
tools/testing/selftests/bpf/prog_tests/rhash.c
View File

@ -172,6 +172,12 @@ void test_rhash(void)
if (test__start_subtest("test_rhash_delete_nonexistent"))
rhash_run("test_rhash_delete_nonexistent");
if (test__start_subtest("test_rhash_kptr_update"))
rhash_run("test_rhash_kptr_update");
if (test__start_subtest("test_rhash_kptr_delete"))
rhash_run("test_rhash_kptr_delete");
if (test__start_subtest("test_rhash_map_extra_presize"))
rhash_map_extra_presize();

141
tools/testing/selftests/bpf/prog_tests/rhash_timer.c Normal file
View File

@ -0,0 +1,141 @@
// SPDX-License-Identifier: GPL-2.0
#define _GNU_SOURCE
#include <sched.h>
#include <test_progs.h>
#include "rhash_timer.skel.h"
#define MAX_ATTEMPTS 256
#define RCU_SYNC_INTERVAL 64
static int pin_to_first_cpu(cpu_set_t *old_mask)
{
cpu_set_t new_mask;
int cpu;
if (sched_getaffinity(0, sizeof(*old_mask), old_mask))
return -errno;
for (cpu = 0; cpu < CPU_SETSIZE; cpu++)
if (CPU_ISSET(cpu, old_mask))
break;
if (cpu == CPU_SETSIZE)
return -EINVAL;
CPU_ZERO(&new_mask);
CPU_SET(cpu, &new_mask);
if (sched_setaffinity(0, sizeof(new_mask), &new_mask))
return -errno;
return 0;
}
static int update_timer_map(int map_fd, __u64 key)
{
__u64 value[3] = {};
return bpf_map_update_elem(map_fd, &key, value, BPF_NOEXIST);
}
static int run_prog(int prog_fd, struct bpf_test_run_opts *opts)
{
int err;
err = bpf_prog_test_run_opts(prog_fd, opts);
if (err)
return err;
return opts->retval;
}
void test_rhash_timer(void)
{
LIBBPF_OPTS(bpf_test_run_opts, opts);
struct rhash_timer *skel = NULL;
cpu_set_t old_mask;
int map_fd = -1, arm_fd, cancel_fd;
bool affinity_set = false;
__u64 key = 1;
int attempt, err;
err = pin_to_first_cpu(&old_mask);
if (!ASSERT_OK(err, "pin_to_first_cpu"))
return;
affinity_set = true;
skel = rhash_timer__open_and_load();
if (!ASSERT_OK_PTR(skel, "open_and_load"))
goto out;
map_fd = bpf_map__fd(skel->maps.timer_map);
if (!ASSERT_GE(map_fd, 0, "timer_map fd"))
goto out;
arm_fd = bpf_program__fd(skel->progs.arm_deleted_timer);
if (!ASSERT_GE(arm_fd, 0, "arm_deleted_timer fd"))
goto out;
cancel_fd = bpf_program__fd(skel->progs.cancel_recycled_timer);
if (!ASSERT_GE(cancel_fd, 0, "cancel_recycled_timer fd"))
goto out;
err = update_timer_map(map_fd, key);
if (!ASSERT_OK(err, "seed_timer_map"))
goto out;
for (attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
err = run_prog(arm_fd, &opts);
if (err) {
ASSERT_OK(err, "arm_deleted_timer");
goto out;
}
if (skel->bss->armed != attempt + 1) {
ASSERT_EQ(skel->bss->armed, attempt + 1, "armed");
goto out;
}
if (skel->bss->timer_init_err) {
ASSERT_OK(skel->bss->timer_init_err, "timer_init_err");
goto out;
}
if (skel->bss->timer_set_callback_err) {
ASSERT_OK(skel->bss->timer_set_callback_err,
"timer_set_callback_err");
goto out;
}
if (skel->bss->timer_start_err) {
ASSERT_OK(skel->bss->timer_start_err, "timer_start_err");
goto out;
}
if ((attempt + 1) % RCU_SYNC_INTERVAL == 0) {
err = kern_sync_rcu();
if (err) {
ASSERT_OK(err, "kern_sync_rcu");
goto out;
}
}
err = update_timer_map(map_fd, ++key);
if (err) {
ASSERT_OK(err, "replace_timer_map");
goto out;
}
err = run_prog(cancel_fd, &opts);
if (err) {
ASSERT_OK(err, "cancel_recycled_timer");
goto out;
}
if (skel->bss->timer_cancel_err) {
ASSERT_OK(skel->bss->timer_cancel_err, "timer_cancel_err");
goto out;
}
if (skel->bss->cancelled)
break;
}
ASSERT_GT(skel->bss->cancelled, 0, "preserved timer");
out:
if (map_fd >= 0)
bpf_map_delete_elem(map_fd, &key);
rhash_timer__destroy(skel);
if (affinity_set)
sched_setaffinity(0, sizeof(old_mask), &old_mask);
}

79
tools/testing/selftests/bpf/prog_tests/snprintf_btf.c
View File

@ -1,7 +1,9 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include <linux/btf.h>
#include <bpf/btf.h>
#include "netif_receive_skb.skel.h"
#include "snprintf_btf_void.skel.h"
/* Demonstrate that bpf_snprintf_btf succeeds and that various data types
* are formatted correctly.
@ -58,3 +60,80 @@ void serial_test_snprintf_btf(void)
cleanup:
netif_receive_skb__destroy(skel);
}
/*
* bpf_snprintf_btf() renders a type_id taken straight from the vmlinux BTF.
* Two such type_ids used to NULL-deref in the BTF show path:
* - a "const void" (a modifier resolving to void) in btf_modifier_show()
* - a BTF_KIND_VAR in btf_var_show() (base BTF has no resolved_ids)
* A fixed kernel renders both without crashing.
*/
static long run(struct snprintf_btf_void *skel, __u32 type_id)
{
LIBBPF_OPTS(bpf_test_run_opts, topts);
char ctx[8] = {};
skel->bss->type_id = type_id;
topts.ctx_in = ctx;
topts.ctx_size_in = sizeof(ctx);
if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.dump_type),
&topts), "test_run"))
return -1;
return skel->bss->ret;
}
void test_snprintf_btf_void(void)
{
const struct btf_type *t;
struct snprintf_btf_void *skel;
int i, n, cv = 0, var = 0;
struct btf *btf;
btf = btf__parse("/sys/kernel/btf/vmlinux", NULL);
if (!btf) {
test__skip();
return;
}
skel = snprintf_btf_void__open_and_load();
if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
goto out_btf;
n = btf__type_cnt(btf);
for (i = 1; i < n && !(cv && var); i++) {
t = btf__type_by_id(btf, i);
if (!cv && btf_kind(t) == BTF_KIND_CONST && t->type == 0)
cv = i;
/* Pick a VAR small enough to render from the program's buffer. */
if (!var && btf_kind(t) == BTF_KIND_VAR) {
long sz = btf__resolve_size(btf, t->type);
if (sz > 0 && sz <= (long)sizeof(skel->bss->obj))
var = i;
}
}
/* "const void" renders the "<unsupported kind:0>" placeholder. */
if (test__start_subtest("const_void")) {
if (cv) {
ASSERT_EQ(run(skel, cv),
sizeof("<unsupported kind:0>") - 1, "ret");
ASSERT_STREQ(skel->bss->out, "<unsupported kind:0>",
"placeholder");
} else {
test__skip();
}
}
/* A BTF_KIND_VAR must resolve and render without error. */
if (test__start_subtest("var")) {
if (var)
ASSERT_GT(run(skel, var), 0, "ret");
else
test__skip();
}
snprintf_btf_void__destroy(skel);
out_btf:
btf__free(btf);
}

29
tools/testing/selftests/bpf/prog_tests/timer_mim.c
View File

@ -59,10 +59,32 @@ void serial_test_timer_mim(void)
int err;
old_print_fn = libbpf_set_print(NULL);
timer_reject_skel = timer_mim_reject__open_and_load();
libbpf_set_print(old_print_fn);
if (!ASSERT_ERR_PTR(timer_reject_skel, "timer_reject_skel_load"))
timer_reject_skel = timer_mim_reject__open();
if (!ASSERT_OK_PTR(timer_reject_skel, "timer_reject_skel_open"))
goto cleanup;
bpf_program__set_autoload(timer_reject_skel->progs.test1, true);
err = timer_mim_reject__load(timer_reject_skel);
ASSERT_ERR(err, "timer_reject_skel_load");
timer_mim_reject__destroy(timer_reject_skel);
timer_reject_skel = timer_mim_reject__open();
if (!ASSERT_OK_PTR(timer_reject_skel, "callback_reject_skel_open"))
goto cleanup;
bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_mismatch, true);
err = timer_mim_reject__load(timer_reject_skel);
ASSERT_ERR(err, "callback_reject_skel_load");
timer_mim_reject__destroy(timer_reject_skel);
timer_reject_skel = timer_mim_reject__open();
if (!ASSERT_OK_PTR(timer_reject_skel, "callback_accept_skel_open"))
goto cleanup;
bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_match, true);
err = timer_mim_reject__load(timer_reject_skel);
if (!ASSERT_OK(err, "callback_accept_skel_load"))
goto cleanup;
timer_mim_reject__destroy(timer_reject_skel);
timer_reject_skel = NULL;
libbpf_set_print(old_print_fn);
timer_skel = timer_mim__open_and_load();
if (!timer_skel && errno == EOPNOTSUPP) {
@ -75,6 +97,7 @@ void serial_test_timer_mim(void)
err = timer_mim(timer_skel);
ASSERT_OK(err, "timer_mim");
cleanup:
libbpf_set_print(old_print_fn);
timer_mim__destroy(timer_skel);
timer_mim_reject__destroy(timer_reject_skel);
}

39
tools/testing/selftests/bpf/progs/iters.c
View File

@ -2149,4 +2149,43 @@ __naked int stack_misc_vs_scalar_in_a_loop(void)
);
}
__used
static int loop_cb5(int i, __u64 *ctx)
{
/* unsafe on a second iteration */
small_arr[*ctx] = i;
*ctx = 100500;
return 0;
}
SEC("raw_tp")
__flag(BPF_F_TEST_STATE_FREQ)
__failure __msg("memory access is {{.*}} and is outside of the object of size 64")
__naked void loop_counter_precision_2nd_iter(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
"*(u64 *)(r10 - 8) = 0;"
"r1 = 2;"
"if r0 == 42 goto +1;"
"r1 = 1;"
"r2 = loop_cb5 ll;"
"r3 = r10;"
"r3 += -8;"
"r4 = 0;"
/*
* Explore with nr_loops=1 on a first path and nr_loops=2 on a second path.
* Buggy verifier did not propagate r1 precision properly,
* and thus checkpoints created for nr_loops=1 case matched nr_loops=2 case.
*/
"call %[bpf_loop];"
"r0 = 0;"
"exit;"
:
: __imm(bpf_loop),
__imm(bpf_get_prandom_u32)
: __clobber_all
);
}
char _license[] SEC("license") = "GPL";

37
tools/testing/selftests/bpf/progs/map_kptr_fail.c
View File

@ -409,4 +409,41 @@ int reject_scalar_store_to_kptr(struct __sk_buff *ctx)
return 0;
}
SEC("?tc")
__description("reject imprecise scalar store to kptr after state pruning")
__failure __msg("invalid kptr access, R7 type=scalar")
__naked void reject_imprecise_scalar_store_to_kptr(void)
{
asm volatile (
"r0 = 0;"
"*(u32 *)(r10 - 4) = r0;"
"r2 = r10;"
"r2 += -4;"
"r1 = %[array_map] ll;"
"call %[bpf_map_lookup_elem];"
"if r0 == 0 goto l2_%=;"
"r6 = r0;"
"r9 = *(u64 *)(r6 + 0);"
"if r9 != 0 goto l0_%=;"
"r7 = 0;"
".rept 10;"
"r5 = 1;"
".endr;"
"goto l1_%=;"
"l0_%=:"
"r7 = 0x4141414141414141 ll;"
".rept 10;"
"r5 = 1;"
".endr;"
"l1_%=:"
"*(u64 *)(r6 + 8) = r7;"
"l2_%=:"
"r0 = 0;"
"exit;"
:
: __imm(bpf_map_lookup_elem),
__imm_addr(array_map)
: __clobber_all);
}
char _license[] SEC("license") = "GPL";

59
tools/testing/selftests/bpf/progs/percpu_alloc_fail.c
View File

@ -33,6 +33,20 @@ struct {
__type(value, struct elem);
} array SEC(".maps");
struct kernel_percpu_elem {
struct task_struct __percpu_kptr *task;
};
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 1);
__type(key, int);
__type(value, struct kernel_percpu_elem);
} kernel_percpu_array SEC(".maps");
struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
void bpf_task_release(struct task_struct *p) __ksym;
long ret;
SEC("?fentry/bpf_fentry_test1")
@ -137,6 +151,51 @@ int BPF_PROG(test_array_map_5)
return 0;
}
SEC("?syscall")
__failure __msg("invalid kptr access, R2 type=trusted_ptr_ expected=ptr_task_struct")
int reject_kernel_ptr_into_percpu_kptr(void *ctx)
{
struct kernel_percpu_elem *e;
struct task_struct *p, *old;
int index = 0;
e = bpf_map_lookup_elem(&kernel_percpu_array, &index);
if (!e)
return 0;
p = bpf_task_from_pid(1);
if (!p)
return 0;
old = bpf_kptr_xchg(&e->task, p);
if (old)
bpf_task_release(old);
return 0;
}
SEC("?fentry.s/bpf_fentry_test1")
__failure __msg("invalid kptr access, R2 type=ptr_ expected=ptr_val_t")
int BPF_PROG(reject_plain_alloc_into_percpu_kptr)
{
struct val_t __percpu_kptr *old;
struct val_t *p;
struct elem *e;
int index = 0;
e = bpf_map_lookup_elem(&array, &index);
if (!e)
return 0;
p = bpf_obj_new(struct val_t);
if (!p)
return 0;
old = bpf_kptr_xchg(&e->pc, p);
if (old)
bpf_percpu_obj_drop(old);
return 0;
}
SEC("?fentry.s/bpf_fentry_test1")
__failure __msg("bpf_percpu_obj_new type ID argument must be of a struct of scalars")
int BPF_PROG(test_array_map_6)

52
tools/testing/selftests/bpf/progs/preempt_lock.c
View File

@ -115,6 +115,58 @@ int preempt_sleepable_helper(void *ctx)
return 0;
}
SEC("?uprobe.s")
__failure __msg("sleepable helper bpf_get_stack#")
int preempt_sleepable_get_stack(struct pt_regs *ctx)
{
struct bpf_stack_build_id stack;
bpf_preempt_disable();
bpf_get_stack(ctx, &stack, sizeof(stack),
BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
bpf_preempt_enable();
return 0;
}
SEC("?uprobe.s")
__failure __msg("sleepable helper bpf_get_task_stack#")
int preempt_sleepable_get_task_stack(void *ctx)
{
struct bpf_stack_build_id stack;
struct task_struct *task;
task = bpf_get_current_task_btf();
bpf_preempt_disable();
bpf_get_task_stack(task, &stack, sizeof(stack),
BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
bpf_preempt_enable();
return 0;
}
SEC("?uprobe.s")
__success
int sleepable_get_stack(struct pt_regs *ctx)
{
struct bpf_stack_build_id stack;
bpf_get_stack(ctx, &stack, sizeof(stack),
BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
return 0;
}
SEC("?uprobe.s")
__success
int sleepable_get_task_stack(void *ctx)
{
struct bpf_stack_build_id stack;
struct task_struct *task;
task = bpf_get_current_task_btf();
bpf_get_task_stack(task, &stack, sizeof(stack),
BPF_F_USER_STACK | BPF_F_USER_BUILD_ID);
return 0;
}
SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
__failure __msg("kernel func bpf_copy_from_user_str is sleepable within non-preemptible region")
int preempt_sleepable_kfunc(void *ctx)

53
tools/testing/selftests/bpf/progs/raw_tp_null_fail.c
View File

@ -22,3 +22,56 @@ int test_raw_tp_null_sched_pi_setprio_arg_2(void *ctx) {
asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u64 *)(r1 +0);" ::: __clobber_all);
return 0;
}
/* Plain raw tracepoint arguments remain scalar values. */
SEC("raw_tp/signal_generate")
__success
int test_raw_tp_signal_generate_info_scalar(void *ctx)
{
asm volatile("r1 = *(u64 *)(r1 +8); if r1 != 1 goto +0;" ::: __clobber_all);
return 0;
}
/* tp_btf programs may inspect the sentinel as a scalar value. */
SEC("tp_btf/signal_generate")
__success
int test_tp_btf_signal_generate_info_scalar(void *ctx)
{
asm volatile("r1 = *(u64 *)(r1 +8); if r1 != 1 goto +0;" ::: __clobber_all);
return 0;
}
/* SEND_SIG_PRIV is non-NULL, so a NULL check cannot make info safe. */
SEC("tp_btf/signal_generate")
__failure __msg("R1 invalid mem access 'scalar'")
int test_tp_btf_signal_generate_info_no_deref(void *ctx)
{
asm volatile("r1 = *(u64 *)(r1 +8); if r1 == 0 goto +1; "
"r1 = *(u32 *)(r1 +0);" ::: __clobber_all);
return 0;
}
SEC("tp_btf/signal_deliver")
__failure __msg("R1 invalid mem access 'scalar'")
int test_tp_btf_signal_deliver_info_no_deref(void *ctx)
{
asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u32 *)(r1 +0);" ::: __clobber_all);
return 0;
}
SEC("tp_btf/sched_process_wait")
__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'")
int test_raw_tp_null_sched_process_wait_arg_1(void *ctx)
{
asm volatile("r1 = *(u64 *)(r1 +0); r1 = *(u32 *)(r1 +0);" ::: __clobber_all);
return 0;
}
SEC("tp_btf/sched_process_wait")
__success
int test_raw_tp_null_sched_process_wait_arg_1_checked(void *ctx)
{
asm volatile("r1 = *(u64 *)(r1 +0); if r1 == 0 goto +1; "
"r1 = *(u32 *)(r1 +0);" ::: __clobber_all);
return 0;
}

84
tools/testing/selftests/bpf/progs/rbtree_fail.c
View File

@ -16,6 +16,7 @@ struct node_data {
private(A) struct bpf_spin_lock glock;
private(A) struct bpf_rb_root groot __contains(node_data, node);
private(A) struct bpf_rb_root groot2 __contains(node_data, node);
private(B) struct bpf_res_spin_lock res_glock;
static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
@ -265,6 +266,53 @@ static bool less__bad_fn_call_first_unlock_after(struct bpf_rb_node *a, const st
return node_a->key < node_b->key;
}
static bool less__bad_res_spin_unlock(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
bpf_res_spin_unlock(&res_glock);
return false;
}
static __noinline void rbtree_cb_unlock_relock(void)
{
bpf_spin_unlock(&glock);
bpf_spin_lock(&glock);
}
static __noinline void rbtree_cb_nested_unlock(void)
{
rbtree_cb_unlock_relock();
asm volatile ("");
}
static bool less__bad_subprog_unlock(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
struct node_data *node_a;
struct node_data *node_b;
node_a = container_of(a, struct node_data, node);
node_b = container_of(b, struct node_data, node);
rbtree_cb_nested_unlock();
return node_a->key < node_b->key;
}
static __noinline void rbtree_cb_noop(void)
{
asm volatile ("");
}
static bool less__subprog_allowed(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
struct node_data *node_a;
struct node_data *node_b;
node_a = container_of(a, struct node_data, node);
node_b = container_of(b, struct node_data, node);
rbtree_cb_noop();
return node_a->key < node_b->key;
}
static __always_inline
long add_with_cb(bool (cb)(struct bpf_rb_node *a, const struct bpf_rb_node *b))
{
@ -301,4 +349,40 @@ long rbtree_api_add_bad_cb_bad_fn_call_first_unlock_after(void *ctx)
return add_with_cb(less__bad_fn_call_first_unlock_after);
}
SEC("?tc")
__failure __msg("can't res_spin_{lock,unlock} in rbtree cb")
long rbtree_api_add_bad_cb_res_spin_unlock(void *ctx)
{
struct node_data *n;
n = bpf_obj_new(typeof(*n));
if (!n)
return 1;
bpf_spin_lock(&glock);
if (bpf_res_spin_lock(&res_glock)) {
bpf_spin_unlock(&glock);
bpf_obj_drop(n);
return 1;
}
bpf_rbtree_add(&groot, &n->node, less__bad_res_spin_unlock);
bpf_res_spin_unlock(&res_glock);
bpf_spin_unlock(&glock);
return 0;
}
SEC("?tc")
__failure __msg("can't spin_{lock,unlock} in rbtree cb")
long rbtree_api_add_bad_cb_subprog_unlock(void *ctx)
{
return add_with_cb(less__bad_subprog_unlock);
}
SEC("?tc")
__success
long rbtree_api_add_cb_subprog_allowed(void *ctx)
{
return add_with_cb(less__subprog_allowed);
}
char _license[] SEC("license") = "GPL";

6
tools/testing/selftests/bpf/progs/rcu_read_lock.c
View File

@ -592,9 +592,9 @@ int non_own_ref_untrusted_ld(void *ctx)
}
bpf_rcu_read_unlock();
/*
* The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED
* | NON_OWN_REF, and the load below has to get the BPF_PROBE_MEM
* rewrite for it, otherwise a bad address panics the kernel.
* The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED,
* and the load below has to get the BPF_PROBE_MEM rewrite for it,
* otherwise a bad address panics the kernel.
*/
non_own_ref_key = node->key;
return 0;

61
tools/testing/selftests/bpf/progs/refcounted_kptr.c
View File

@ -23,6 +23,15 @@ struct map_value {
struct node_data __kptr *node;
};
struct node_refcount_only {
long key;
struct bpf_refcount refcount;
};
struct map_value_refcount_only {
struct node_refcount_only __kptr *node;
};
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, int);
@ -30,6 +39,13 @@ struct {
__uint(max_entries, 2);
} stashed_nodes SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, int);
__type(value, struct map_value_refcount_only);
__uint(max_entries, 1);
} stashed_refcount_only SEC(".maps");
struct node_acquire {
long key;
long data;
@ -832,6 +848,51 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx)
return 0;
}
SEC("tc")
__success
long refcount_acquire_owning_input_no_null_check(void *ctx)
{
struct node_refcount_only *n, *m;
n = bpf_obj_new(typeof(*n));
if (!n)
return 1;
m = bpf_refcount_acquire(n);
bpf_obj_drop(m);
bpf_obj_drop(n);
return 0;
}
SEC("?syscall")
__success
long refcount_acquire_rcu_map_kptr_null_checked(void *ctx)
{
struct map_value_refcount_only *mapval;
struct node_refcount_only *n, *m;
int idx = 0;
mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx);
if (!mapval)
return 1;
bpf_rcu_read_lock();
n = mapval->node;
if (!n) {
bpf_rcu_read_unlock();
return 2;
}
m = bpf_refcount_acquire(n);
bpf_rcu_read_unlock();
if (!m)
return 3;
bpf_obj_drop(m);
return 0;
}
static long __stash_map_empty_xchg(struct node_data *n, int idx)
{
struct map_value *mapval = bpf_map_lookup_elem(&stashed_nodes, &idx);

150
tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c
View File

@ -19,6 +19,26 @@ struct node_refcounted {
struct bpf_refcount refcount;
};
struct node_refcount_only {
long key;
struct bpf_refcount refcount;
};
struct map_value_refcount_only {
struct node_refcount_only __kptr *node;
};
struct rcu_graph_node {
struct bpf_rb_node node;
long data;
};
struct rcu_graph_node *just_here_because_btf_bug;
struct map_value_rcu_graph {
struct rcu_graph_node __kptr *node;
};
extern void bpf_rcu_read_lock(void) __ksym;
extern void bpf_rcu_read_unlock(void) __ksym;
@ -27,6 +47,22 @@ private(A) struct bpf_spin_lock glock;
private(A) struct bpf_rb_root groot __contains(node_acquire, node);
private(B) struct bpf_spin_lock lock;
private(B) struct bpf_list_head head __contains(node_refcounted, list);
private(C) struct bpf_spin_lock graph_lock;
private(C) struct bpf_rb_root graph_root __contains(rcu_graph_node, node);
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, int);
__type(value, struct map_value_refcount_only);
__uint(max_entries, 1);
} stashed_refcount_only SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, int);
__type(value, struct map_value_rcu_graph);
__uint(max_entries, 1);
} stashed_rcu_graph SEC(".maps");
static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b)
{
@ -89,6 +125,120 @@ long refcount_acquire_non_object(void *ctx)
return bpf_refcount_acquire(ctx) != NULL;
}
SEC("?syscall")
__failure __msg("Possibly NULL pointer passed to trusted R1")
long refcount_acquire_rcu_map_kptr_unchecked_drop(void *ctx)
{
struct map_value_refcount_only *mapval;
struct node_refcount_only *tmp, *n, *m;
int idx = 0;
/* Force Clang to emit complete BTF for struct node_refcount_only. */
tmp = bpf_obj_new(typeof(*tmp));
if (!tmp)
return 3;
bpf_obj_drop(tmp);
mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx);
if (!mapval)
return 1;
bpf_rcu_read_lock();
n = mapval->node;
if (!n) {
bpf_rcu_read_unlock();
return 2;
}
m = bpf_refcount_acquire(n);
bpf_rcu_read_unlock();
bpf_obj_drop(m);
return 0;
}
SEC("?syscall")
__failure
__msg("bpf_rbtree_remove can only take non-owning or refcounted "
"bpf_rb_node pointer")
long rbtree_remove_after_rcu_unlock(void *ctx)
{
struct map_value_rcu_graph *mapval;
struct bpf_rb_node *rb_node;
struct rcu_graph_node *node;
int idx = 0;
mapval = bpf_map_lookup_elem(&stashed_rcu_graph, &idx);
if (!mapval)
return 0;
bpf_rcu_read_lock();
node = mapval->node;
if (!node) {
bpf_rcu_read_unlock();
return 0;
}
bpf_rcu_read_unlock();
bpf_spin_lock(&graph_lock);
rb_node = bpf_rbtree_remove(&graph_root, &node->node);
bpf_spin_unlock(&graph_lock);
if (rb_node)
bpf_obj_drop(container_of(rb_node, struct rcu_graph_node, node));
return 0;
}
SEC("?syscall")
__failure __msg("R1 is neither owning or non-owning ref")
long refcount_acquire_after_rcu_unlock(void *ctx)
{
struct map_value_refcount_only *mapval;
struct node_refcount_only *node, *ref;
int idx = 0;
mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx);
if (!mapval)
return 0;
bpf_rcu_read_lock();
node = mapval->node;
if (!node) {
bpf_rcu_read_unlock();
return 0;
}
bpf_rcu_read_unlock();
ref = bpf_refcount_acquire(node);
if (ref)
bpf_obj_drop(ref);
return 0;
}
SEC("?syscall")
__failure __msg("invalid mem access 'scalar'")
long graph_kptr_after_spin_unlock(void *ctx)
{
struct map_value_rcu_graph *mapval;
struct rcu_graph_node *node;
int idx = 0;
mapval = bpf_map_lookup_elem(&stashed_rcu_graph, &idx);
if (!mapval)
return 0;
bpf_spin_lock(&graph_lock);
node = mapval->node;
if (!node) {
bpf_spin_unlock(&graph_lock);
return 0;
}
bpf_spin_unlock(&graph_lock);
return node->data;
}
SEC("?tc")
__failure __msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}")
long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx)

112
tools/testing/selftests/bpf/progs/rhash.c
View File

@ -19,6 +19,11 @@ struct elem {
int val;
};
struct special_elem {
struct task_struct __kptr *task;
int val;
};
struct {
__uint(type, BPF_MAP_TYPE_RHASH);
__uint(map_flags, BPF_F_NO_PREALLOC);
@ -27,6 +32,17 @@ struct {
__type(value, struct elem);
} rhmap SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_RHASH);
__uint(map_flags, BPF_F_NO_PREALLOC);
__uint(max_entries, 1);
__type(key, int);
__type(value, struct special_elem);
} special_fields SEC(".maps");
extern struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
extern void bpf_task_release(struct task_struct *p) __ksym;
SEC("syscall")
int test_rhash_lookup_update(void *ctx)
{
@ -246,3 +262,99 @@ int test_rhash_delete_nonexistent(void *ctx)
err = 0;
return 0;
}
SEC("syscall")
int test_rhash_kptr_update(void *ctx)
{
struct special_elem val1 = { .val = 1 };
struct special_elem val2 = { .val = 2 };
struct task_struct *task, *old;
struct special_elem *elem;
int key = 0;
err = 1;
if (bpf_map_update_elem(&special_fields, &key, &val1, BPF_NOEXIST))
return 1;
err = 2;
elem = bpf_map_lookup_elem(&special_fields, &key);
if (!elem)
return 2;
err = 3;
task = bpf_task_acquire(bpf_get_current_task_btf());
if (!task)
return 3;
err = 4;
old = bpf_kptr_xchg(&elem->task, task);
if (old) {
bpf_task_release(old);
return 4;
}
err = 5;
if (bpf_map_update_elem(&special_fields, &key, &val2, BPF_EXIST))
return 5;
err = 6;
elem = bpf_map_lookup_elem(&special_fields, &key);
if (!elem || elem->val != 2)
return 6;
err = 7;
old = bpf_kptr_xchg(&elem->task, NULL);
if (!old)
return 7;
bpf_task_release(old);
err = 8;
if (bpf_map_delete_elem(&special_fields, &key))
return 8;
err = 0;
return 0;
}
SEC("syscall")
int test_rhash_kptr_delete(void *ctx)
{
struct special_elem val = {};
struct task_struct *task, *old;
struct special_elem *elem;
int key = 0;
err = 1;
if (bpf_map_update_elem(&special_fields, &key, &val, BPF_NOEXIST))
return 1;
err = 2;
elem = bpf_map_lookup_elem(&special_fields, &key);
if (!elem)
return 2;
err = 3;
task = bpf_task_acquire(bpf_get_current_task_btf());
if (!task)
return 3;
err = 4;
old = bpf_kptr_xchg(&elem->task, task);
if (old) {
bpf_task_release(old);
return 4;
}
err = 5;
if (bpf_map_delete_elem(&special_fields, &key))
return 5;
err = 6;
old = bpf_kptr_xchg(&elem->task, NULL);
if (!old)
return 6;
bpf_task_release(old);
err = 0;
return 0;
}

98
tools/testing/selftests/bpf/progs/rhash_timer.c Normal file
View File

@ -0,0 +1,98 @@
// SPDX-License-Identifier: GPL-2.0
#include <vmlinux.h>
#include <errno.h>
#include <bpf/bpf_helpers.h>
#define CLOCK_MONOTONIC 1
#define TIMER_NSEC (60ULL * 1000 * 1000 * 1000)
struct timer_value {
struct bpf_timer timer;
u64 data;
};
struct {
__uint(type, BPF_MAP_TYPE_RHASH);
__uint(map_flags, BPF_F_NO_PREALLOC);
__uint(max_entries, 1);
__type(key, u64);
__type(value, struct timer_value);
} timer_map SEC(".maps");
u64 armed;
u64 cancelled;
long timer_init_err;
long timer_set_callback_err;
long timer_start_err;
long timer_cancel_err;
static int timer_cb(void *map, u64 *key, struct timer_value *value)
{
return 0;
}
static long arm_timer_cb(struct bpf_map *map, u64 *key,
struct timer_value *value, void *ctx)
{
u64 key_copy = *key;
long err;
err = bpf_map_delete_elem(map, &key_copy);
if (err)
return 1;
err = bpf_timer_init(&value->timer, map, CLOCK_MONOTONIC);
if (err) {
timer_init_err = err;
return 1;
}
err = bpf_timer_set_callback(&value->timer, timer_cb);
if (err) {
timer_set_callback_err = err;
return 1;
}
err = bpf_timer_start(&value->timer, TIMER_NSEC, BPF_F_TIMER_CPU_PIN);
if (err) {
timer_start_err = err;
return 1;
}
__sync_fetch_and_add(&armed, 1);
return 1;
}
static long cancel_timer_cb(struct bpf_map *map, u64 *key,
struct timer_value *value, void *ctx)
{
long err;
err = bpf_timer_cancel(&value->timer);
if (err == -EINVAL)
return 1;
if (err < 0) {
timer_cancel_err = err;
return 1;
}
__sync_fetch_and_add(&cancelled, 1);
return 1;
}
SEC("syscall")
int arm_deleted_timer(void *ctx)
{
bpf_for_each_map_elem(&timer_map, arm_timer_cb, NULL, 0);
return 0;
}
SEC("syscall")
int cancel_recycled_timer(void *ctx)
{
bpf_for_each_map_elem(&timer_map, cancel_timer_cb, NULL, 0);
return 0;
}
char _license[] SEC("license") = "GPL";

24
tools/testing/selftests/bpf/progs/snprintf_btf_void.c Normal file
View File

@ -0,0 +1,24 @@
// SPDX-License-Identifier: GPL-2.0
#include "btf_ptr.h"
#include <bpf/bpf_helpers.h>
__u32 type_id;
/* A buffer we own to render the selected type from, kept in bounds. */
char obj[256];
char out[64];
long ret;
SEC("raw_tp/sys_enter")
int dump_type(void *ctx)
{
struct btf_ptr ptr = {
.ptr = obj,
.type_id = type_id,
.flags = 0,
};
ret = bpf_snprintf_btf(out, sizeof(out), &ptr, sizeof(ptr), 0);
return 0;
}
char _license[] SEC("license") = "GPL";

16
tools/testing/selftests/bpf/progs/tailcall_callback.c
View File

@ -44,6 +44,13 @@ int callback_loop(int index, void **cb_ctx)
return ret ? 1 : 0;
}
static __noinline
int callback_tail(int index, void **cb_ctx)
{
bpf_tail_call_static(*cb_ctx, &jmp_table, 0);
return 0;
}
static __noinline
int callback_empty(int index, void *data)
{
@ -78,4 +85,13 @@ int tailcall_callback_2(struct __sk_buff *skb)
return 0;
}
/* callback with a direct tail call is rejected without a verifier bug */
SEC("tc")
__failure __msg("cannot tail call within callback")
int tailcall_callback_3(struct __sk_buff *skb)
{
bpf_loop(1, callback_tail, &skb, 0);
return 0;
}
char __license[] SEC("license") = "GPL";

3
tools/testing/selftests/bpf/progs/task_local_data.bpf.h
View File

@ -61,6 +61,7 @@
#define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1)
#define TLD_MAX_DATA_CNT (__PAGE_SIZE / sizeof(struct tld_metadata) - 1)
#define TLD_DATA_SIZE (__PAGE_SIZE - sizeof(__u64))
#ifndef TLD_NAME_LEN
#define TLD_NAME_LEN 62
@ -189,6 +190,8 @@ static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_s
return start + off;
off += TLD_ROUND_UP(metadata[i].size, 8);
if (off > TLD_DATA_SIZE)
break;
}
return -cnt;

8
tools/testing/selftests/bpf/progs/test_bpf_nf.c
View File

@ -190,8 +190,8 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32,
ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def,
sizeof(opts_def));
if (ct) {
__u16 sport = bpf_get_prandom_u32();
__u16 dport = bpf_get_prandom_u32();
__u16 sport = bpf_get_prandom_u32() % 65535 + 1;
__u16 dport = bpf_get_prandom_u32() % 65535 + 1;
union nf_inet_addr saddr = {};
union nf_inet_addr daddr = {};
struct nf_conn *ct_ins;
@ -293,8 +293,8 @@ nf_ct_opts_new_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *
ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def,
sizeof(opts_def));
if (ct) {
__u16 sport = bpf_get_prandom_u32();
__u16 dport = bpf_get_prandom_u32();
__u16 sport = bpf_get_prandom_u32() % 65535 + 1;
__u16 dport = bpf_get_prandom_u32() % 65535 + 1;
union nf_inet_addr saddr = {};
union nf_inet_addr daddr = {};
struct nf_conn *ct_ins;

84
tools/testing/selftests/bpf/progs/timer_mim_reject.c
View File

@ -43,7 +43,7 @@ static int timer_cb(void *map, int *key, struct hmap_elem *val)
return 0;
}
SEC("fentry/bpf_fentry_test1")
SEC("?fentry/bpf_fentry_test1")
int BPF_PROG(test1, int a)
{
struct hmap_elem init = {};
@ -72,3 +72,85 @@ int BPF_PROG(test1, int a)
err |= 8;
return 0;
}
struct callback_ctx {
void *map;
};
static int mismatch_iter_cb(void *map, int *key, struct hmap_elem *val, struct callback_ctx *ctx)
{
bpf_timer_init(&val->timer, ctx->map, CLOCK_MONOTONIC);
return 0;
}
static int timer_mismatch_cb(void *map, int *key, struct hmap_elem *val)
{
struct callback_ctx ctx = { .map = map };
struct bpf_map *inner_map2;
int array_key2 = ARRAY_KEY2;
inner_map2 = bpf_map_lookup_elem(&outer_arr, &array_key2);
if (!inner_map2)
return 0;
bpf_for_each_map_elem(inner_map2, mismatch_iter_cb, &ctx, 0);
return 0;
}
static int match_iter_cb(void *map, int *key, struct hmap_elem *val, struct callback_ctx *ctx)
{
bpf_timer_init(&val->timer, map, CLOCK_MONOTONIC);
return 0;
}
static int timer_match_cb(void *map, int *key, struct hmap_elem *val)
{
struct callback_ctx ctx = {};
struct bpf_map *inner_map2;
int array_key2 = ARRAY_KEY2;
inner_map2 = bpf_map_lookup_elem(&outer_arr, &array_key2);
if (!inner_map2)
return 0;
bpf_for_each_map_elem(inner_map2, match_iter_cb, &ctx, 0);
return 0;
}
SEC("?fentry/bpf_fentry_test1")
int BPF_PROG(callback_map_uid_mismatch, int a)
{
struct hmap_elem *val;
struct bpf_map *inner_map;
int array_key = ARRAY_KEY;
int hash_key = HASH_KEY;
inner_map = bpf_map_lookup_elem(&outer_arr, &array_key);
if (!inner_map)
return 0;
val = bpf_map_lookup_elem(inner_map, &hash_key);
if (!val)
return 0;
bpf_timer_init(&val->timer, inner_map, CLOCK_MONOTONIC);
bpf_timer_set_callback(&val->timer, timer_mismatch_cb);
return 0;
}
SEC("?fentry/bpf_fentry_test1")
int BPF_PROG(callback_map_uid_match, int a)
{
struct hmap_elem *val;
struct bpf_map *inner_map;
int array_key = ARRAY_KEY;
int hash_key = HASH_KEY;
inner_map = bpf_map_lookup_elem(&outer_arr, &array_key);
if (!inner_map)
return 0;
val = bpf_map_lookup_elem(inner_map, &hash_key);
if (!val)
return 0;
bpf_timer_init(&val->timer, inner_map, CLOCK_MONOTONIC);
bpf_timer_set_callback(&val->timer, timer_match_cb);
return 0;
}

98
tools/testing/selftests/bpf/progs/verifier_async_cb_context.c
View File

@ -62,6 +62,104 @@ int timer_sleepable_prog(void *ctx)
return 0;
}
static int timer_sys_bpf_cb(void *map, int *key, struct bpf_timer *timer)
{
__u64 attr = 0;
bpf_sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr));
return 0;
}
SEC("syscall")
__failure __msg("sleepable helper bpf_sys_bpf#{{[0-9]+}} in non-sleepable prog")
int timer_sys_bpf_prog(void *ctx)
{
struct timer_elem *val;
int key = 0;
val = bpf_map_lookup_elem(&timer_map, &key);
if (!val)
return 0;
bpf_timer_init(&val->t, &timer_map, 0);
bpf_timer_set_callback(&val->t, timer_sys_bpf_cb);
return 0;
}
static int timer_sys_close_cb(void *map, int *key, struct bpf_timer *timer)
{
bpf_sys_close(0);
return 0;
}
SEC("syscall")
__failure __msg("sleepable helper bpf_sys_close#{{[0-9]+}} in non-sleepable prog")
int timer_sys_close_prog(void *ctx)
{
struct timer_elem *val;
int key = 0;
val = bpf_map_lookup_elem(&timer_map, &key);
if (!val)
return 0;
bpf_timer_init(&val->t, &timer_map, 0);
bpf_timer_set_callback(&val->t, timer_sys_close_cb);
return 0;
}
static int timer_btf_find_cb(void *map, int *key, struct bpf_timer *timer)
{
char name[] = "task_struct";
bpf_btf_find_by_name_kind(name, sizeof(name), BTF_KIND_STRUCT, 0);
return 0;
}
SEC("syscall")
__failure __msg("sleepable helper bpf_btf_find_by_name_kind#{{[0-9]+}} in non-sleepable prog")
int timer_btf_find_prog(void *ctx)
{
struct timer_elem *val;
int key = 0;
val = bpf_map_lookup_elem(&timer_map, &key);
if (!val)
return 0;
bpf_timer_init(&val->t, &timer_map, 0);
bpf_timer_set_callback(&val->t, timer_btf_find_cb);
return 0;
}
SEC("syscall")
__success
int syscall_sys_bpf_prog(void *ctx)
{
__u64 attr = 0;
bpf_sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr));
return 0;
}
SEC("syscall")
__success
int syscall_sys_close_prog(void *ctx)
{
bpf_sys_close(0);
return 0;
}
SEC("syscall")
__success
int syscall_btf_find_prog(void *ctx)
{
char name[] = "task_struct";
bpf_btf_find_by_name_kind(name, sizeof(name), BTF_KIND_STRUCT, 0);
return 0;
}
/* Workqueue tests */
struct wq_elem {

41
tools/testing/selftests/bpf/progs/verifier_bounds.c
View File

@ -2267,6 +2267,47 @@ __naked void deduce64_from_32_wrapping_32bit(void)
: __clobber_all);
}
/*
* Unprivileged variable pointer arithmetic on a PTR_TO_MAP_VALUE whose
* offset collapses to a constant. The Spectre-v1 speculative path snapshots
* the pointer while its r32 has just been blanked but its offset not yet
* synced; the following register move used to trip reg_bounds_sanity_check()
* ("const subreg tnum out of sync with range bounds"). With
* BPF_F_TEST_REG_INVARIANTS that violation turns into a load failure, so the
* unprivileged program must still load.
*/
SEC("socket")
__success __success_unpriv
__flag(BPF_F_TEST_REG_INVARIANTS)
__naked void spec_ptr_alu_const_offset(void)
{
asm volatile (" \
call %[bpf_ktime_get_ns]; \
*(u64*)(r10 - 16) = r0; \
r1 = 0; \
*(u64*)(r10 - 8) = r1; \
r2 = r10; \
r2 += -8; \
r1 = %[map_hash_8b] ll; \
call %[bpf_map_lookup_elem]; \
if r0 == 0 goto l0_%=; \
r1 = *(u64*)(r10 - 16); \
r2 = 0x40000000; \
if r1 > r2 goto l0_%=; \
if r1 s> 1 goto l0_%=; /* r1 in [0, 1] */ \
r0 += r1; /* ptr += bounded scalar */ \
r9 = r0; /* used to trip the warning */ \
*(u8*)(r0 + 0) = r1; \
l0_%=: r0 = 0; \
exit; \
"
:
: __imm(bpf_ktime_get_ns),
__imm(bpf_map_lookup_elem),
__imm_addr(map_hash_8b)
: __clobber_all);
}
/* Check that range_within() compares cnum ranges, not min/max projections. */
SEC("socket")
__failure __msg("div by zero")

110
tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c
View File

@ -621,6 +621,116 @@ __naked void helper_call_does_not_prevent_bpf_fastcall(void)
: __clobber_all);
}
/* A jump to the first spill executes the whole pattern, rewrite is safe. */
SEC("raw_tp")
__arch_x86_64
__log_level(4)
__msg("subprog 0 (jump_to_first_spill) main {{.*}} stack 0")
__xlated("2: if r0 == 0x2a goto pc+0")
__xlated("3: r0 = ")
__xlated("4: r0 = &(void __percpu *)(r0)")
__success
__naked void jump_to_first_spill(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
"r1 = 1;"
"if r0 == 42 goto l0_%=;"
"l0_%=:"
"*(u64 *)(r10 - 8) = r1;"
"call %[bpf_get_smp_processor_id];"
"r1 = *(u64 *)(r10 - 8);"
"exit;"
:
: __imm(bpf_get_prandom_u32),
__imm(bpf_get_smp_processor_id)
: __clobber_all);
}
/* A jump to the call skips the spill, the pattern must be kept. */
SEC("raw_tp")
__arch_x86_64
__log_level(4)
__msg("subprog 0 (jump_to_call) main {{.*}} stack 8")
__xlated("2: if r0 == 0x2a goto pc+1")
__xlated("3: *(u64 *)(r10 -8) = r1")
__xlated("...")
__xlated("7: r1 = *(u64 *)(r10 -8)")
__success
__naked void jump_to_call(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
"r1 = 1;"
"if r0 == 42 goto l0_%=;"
"*(u64 *)(r10 - 8) = r1;"
"l0_%=:"
"call %[bpf_get_smp_processor_id];"
"r1 = *(u64 *)(r10 - 8);"
"exit;"
:
: __imm(bpf_get_prandom_u32),
__imm(bpf_get_smp_processor_id)
: __clobber_all);
}
/* A jump to the fill skips the spill, the pattern must be kept. */
SEC("raw_tp")
__arch_x86_64
__log_level(4)
__msg("subprog 0 (jump_to_fill) main {{.*}} stack 8")
__xlated("2: if r0 == 0x2a goto pc+4")
__xlated("3: *(u64 *)(r10 -8) = r1")
__xlated("...")
__xlated("7: r1 = *(u64 *)(r10 -8)")
__success
__naked void jump_to_fill(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
"r1 = 1;"
"if r0 == 42 goto l0_%=;"
"*(u64 *)(r10 - 8) = r1;"
"call %[bpf_get_smp_processor_id];"
"l0_%=:"
"r1 = *(u64 *)(r10 - 8);"
"exit;"
:
: __imm(bpf_get_prandom_u32),
__imm(bpf_get_smp_processor_id)
: __clobber_all);
}
/* Same as above, but the fill is entered by an unconditional jump. */
SEC("raw_tp")
__arch_x86_64
__log_level(4)
__msg("subprog 0 (unconditional_jump_to_fill) main {{.*}} stack 8")
__xlated("3: *(u64 *)(r10 -8) = r1")
__xlated("...")
__xlated("7: r1 = *(u64 *)(r10 -8)")
__xlated("8: exit")
__xlated("9: goto pc-3")
__success
__naked void unconditional_jump_to_fill(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
"r1 = 1;"
"if r0 == 42 goto l1_%=;"
"*(u64 *)(r10 - 8) = r1;"
"call %[bpf_get_smp_processor_id];"
"l0_%=:"
"r1 = *(u64 *)(r10 - 8);"
"exit;"
"l1_%=:"
"goto l0_%=;"
:
: __imm(bpf_get_prandom_u32),
__imm(bpf_get_smp_processor_id)
: __clobber_all);
}
SEC("raw_tp")
__arch_x86_64
__log_level(4)

14
tools/testing/selftests/bpf/progs/verifier_cfg.c
View File

@ -3,6 +3,7 @@
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
SEC("socket")
@ -55,6 +56,19 @@ __naked void out_of_range_jump2(void)
" ::: __clobber_all);
}
SEC("socket")
__description("invalid DW LDSX instruction in diagnostics")
__failure __msg("BUG_ldx_99")
__log_level(2)
__naked void invalid_dw_ldsx(void)
{
asm volatile (" \
.8byte %[ldsx_dw]; \
" :
: __imm_insn(ldsx_dw, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_DW, BPF_REG_0, BPF_REG_0, 0, 0))
: __clobber_all);
}
SEC("socket")
__description("loop (back-edge)")
__failure __msg("unreachable insn 1")

29
tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c
View File

@ -305,4 +305,33 @@ __naked void cpu_cgroup_storage_access_6(void)
: __clobber_all);
}
/*
* Verification takes two paths: with r2 being scalar zero on path (1)
* and with r2 being some other scalar on path (2).
* Check that the verifier does not use checkpoints created
* on path (1) to prune path (2).
*/
SEC("cgroup/skb")
__failure
__flag(BPF_F_TEST_STATE_FREQ)
__msg("get_local_storage() doesn't support non-zero flags")
__naked void non_zero_flags_on_a_pruned_path(void)
{
asm volatile (" \
call %[bpf_get_prandom_u32]; \
/* r2 is 0 on the path explored first, 1 on the other */\
r2 = 1; \
if r0 == 0 goto 1f; \
r2 = 0; \
1: r1 = %[cgroup_storage] ll; \
call %[bpf_get_local_storage]; \
r0 = 0; \
exit; \
" :
: __imm(bpf_get_prandom_u32),
__imm(bpf_get_local_storage),
__imm_addr(cgroup_storage)
: __clobber_all);
}
char _license[] SEC("license") = "GPL";

24
tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c
View File

@ -56,6 +56,30 @@ int trusted_task_arg_nullable(void *ctx)
return res;
}
/*
* Check that the verifier does not use checkpoints created
* on path with r1 == 0 to prune path with r1 != 0.
*/
SEC("?tp_btf/task_newtask")
__failure
__flag(BPF_F_TEST_STATE_FREQ)
__msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_")
__naked int null_btf_id_arg_global_subprog(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
"r1 = 42;"
"if r0 > 42 goto 1f;"
"r1 = 0;"
"1:"
"call subprog_trusted_task_nullable;"
"r0 = 0;"
"exit;"
:
: __imm(bpf_get_prandom_u32)
: __clobber_all);
}
__weak int subprog_trusted_task_nonnull(struct task_struct *task __arg_trusted)
{
return task->pid + task->tgid;

17
tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c
View File

@ -168,6 +168,23 @@ static int iter_limit_cb(__u32 idx, struct num_context *ctx)
return 0;
}
SEC("?raw_tp")
__failure __msg("R1 type=ctx expected=scalar")
__naked void bpf_loop_reject_pointer(void)
{
asm volatile (
"r2 = %[iter_limit_cb];"
"r3 = 0;"
"r4 = 0;"
"call %[bpf_loop];"
"exit;"
:
: __imm_ptr(iter_limit_cb),
__imm(bpf_loop)
: __clobber_common
);
}
SEC("?raw_tp")
__success
int bpf_loop_iter_limit_ok(void *unused)

193
tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c
View File

@ -3,7 +3,9 @@
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <stdbool.h>
#include "bpf_misc.h"
#include "bpf_kfuncs.h"
struct {
__uint(type, BPF_MAP_TYPE_XSKMAP);
@ -12,6 +14,13 @@ struct {
__type(value, int);
} map_xskmap SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 1);
__type(key, int);
__type(value, int);
} map_hash SEC(".maps");
/* This is equivalent to the following program:
*
* r6 = skb->sk;
@ -264,4 +273,188 @@ __naked void jne_reg_reg_null_check(void)
: __clobber_all);
}
/*
* A comparison between PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED and
* PTR_TO_MAP_VALUE_OR_NULL should not infer that map pointer is not null.
* A bug in check_cond_jmp_op() made such inference possible.
*/
SEC("raw_tp")
__failure
__msg("error: invalid dereference of R0 (a nullable map value pointer)")
__msg(">>> 11 | (61) r0 = *(u32 *)(r0 +0)")
__naked void untrusted_mem_does_not_infer_map_value_non_null(void)
{
asm volatile (" \
/* r6 = bpf_rdonly_cast(0, 0); */ \
r1 = 0; \
r2 = 0; \
call %[bpf_rdonly_cast]; \
r6 = r0; \
/* r0 = bpf_map_lookup_elem(map_hash, &key); */ \
*(u64 *)(r10 - 8) = 0; \
r1 = %[map_hash] ll; \
r2 = r10; \
r2 += -8; \
call %[bpf_map_lookup_elem]; \
/* \
* buggy verifier assumed that r6 can't be null \
* and marked r0 non-null as well. \
*/ \
if r6 != r0 goto 1f; \
r0 = *(u32 *)(r0 + 0); \
1: r0 = 0; \
exit; \
" :
: __imm(bpf_rdonly_cast),
__imm(bpf_map_lookup_elem),
__imm_addr(map_hash)
: __clobber_all);
}
/*
* A pointer with an offset that is not bounded from above may be null at
* runtime, hence it is not a witness for the pointer it is compared with.
*/
SEC("socket")
__failure
__msg("error: invalid dereference of R7 (a nullable map value pointer)")
__naked void unbounded_offset_does_not_infer_map_value_non_null(void)
{
asm volatile (" \
/* r6 = bpf_map_lookup_elem(map_hash, &0); */ \
*(u64 *)(r10 - 8) = 0; \
r1 = %[map_hash] ll; \
r2 = r10; \
r2 += -8; \
call %[bpf_map_lookup_elem]; \
if r0 == 0 goto 1f; \
r6 = r0; \
/* r7 = bpf_map_lookup_elem(map_hash, &1); */ \
*(u64 *)(r10 - 8) = 1; \
r1 = %[map_hash] ll; \
r2 = r10; \
r2 += -8; \
call %[bpf_map_lookup_elem]; \
r7 = r0; \
/* pointer - pointer is an unknown scalar */ \
r8 = r7; \
r8 -= r6; \
/* r8 is in [0, S64_MAX] */ \
r8 <<= 1; \
r8 >>= 1; \
/* r6 may wrap to zero at runtime */ \
r6 += r8; \
if r7 != r6 goto 1f; \
r0 = *(u8 *)(r7 + 0); \
1: r0 = 0; \
exit; \
" :
: __imm(bpf_map_lookup_elem),
__imm_addr(map_hash)
: __clobber_all);
}
/* Same, but the offset is bounded, so the inference is still done. */
SEC("socket")
__success
__naked void bounded_offset_infers_map_value_non_null(void)
{
asm volatile (" \
/* r6 = bpf_map_lookup_elem(map_hash, &0); */ \
*(u64 *)(r10 - 8) = 0; \
r1 = %[map_hash] ll; \
r2 = r10; \
r2 += -8; \
call %[bpf_map_lookup_elem]; \
if r0 == 0 goto 1f; \
r6 = r0; \
/* r7 = bpf_map_lookup_elem(map_hash, &1); */ \
*(u64 *)(r10 - 8) = 1; \
r1 = %[map_hash] ll; \
r2 = r10; \
r2 += -8; \
call %[bpf_map_lookup_elem]; \
r7 = r0; \
/* pointer - pointer is an unknown scalar */ \
r8 = r7; \
r8 -= r6; \
/* r8 is in [0, 3] */ \
r8 &= 3; \
r6 += r8; \
if r7 != r6 goto 1f; \
r0 = *(u8 *)(r7 + 0); \
1: r0 = 0; \
exit; \
" :
: __imm(bpf_map_lookup_elem),
__imm_addr(map_hash)
: __clobber_all);
}
/*
* The low 32 bits of a map value pointer may be zero, hence a 32-bit
* compare with zero cannot be predicted from the pointer being non-NULL
* and both successors of such a jump have to be verified.
*/
SEC("socket")
__failure __msg("invalid access to map value, value_size=4 off=32 size=4")
__naked void jmp32_ptr_vs_zero_jne(void)
{
asm volatile (" \
/* r0 = bpf_map_lookup_elem(map_hash, &key); */ \
*(u64 *)(r10 - 8) = 0; \
r1 = %[map_hash] ll; \
r2 = r10; \
r2 += -8; \
call %[bpf_map_lookup_elem]; \
if r0 == 0 goto 1f; \
if w0 != 0 goto 1f; \
r0 = *(u32 *)(r0 + 32); \
1: r0 = 0; \
exit; \
" :
: __imm(bpf_map_lookup_elem),
__imm_addr(map_hash)
: __clobber_all);
}
/*
* The below program is explored in two paths: r6 == 0 and r6 == 1.
* On the first path comparison "if r0 == r6 goto 2f" should mark r6 as precise,
* otherwise unsafe path with r6 == 1 would be incorrectly pruned.
*/
SEC("socket")
__failure
__flag(BPF_F_TEST_STATE_FREQ)
__msg("error: invalid dereference of R0 (a nullable map value pointer)")
__naked void imprecise_zero_does_not_infer_map_value_non_null(void)
{
asm volatile (" \
call %[bpf_get_prandom_u32]; \
/* r6 is 0 on the path explored first, 1 on the other */\
r6 = 1; \
if r0 == 0 goto 1f; \
r6 = 0; \
/* r0 = bpf_map_lookup_elem(map_hash, &0); */ \
1: *(u64 *)(r10 - 8) = 0; \
r1 = %[map_hash] ll; \
r2 = r10; \
r2 += -8; \
call %[bpf_map_lookup_elem]; \
if r0 == r6 goto 2f; \
r0 = *(u8 *)(r0 + 0); \
2: r0 = 0; \
exit; \
" :
: __imm(bpf_get_prandom_u32),
__imm(bpf_map_lookup_elem),
__imm_addr(map_hash)
: __clobber_all);
}
void kfunc_root(void)
{
bpf_rdonly_cast(0, 0);
}
char _license[] SEC("license") = "GPL";

96
tools/testing/selftests/bpf/progs/verifier_ld_ind.c
View File

@ -194,6 +194,102 @@ __naked void ld_ind_subprog_both_paths_safe(void)
::: __clobber_all);
}
__naked __noinline __used
static int ld_abs_callback(void)
{
asm volatile (
"r6 = *(u64 *)(r2 + 0);"
".8byte %[ld_abs];"
"r0 = 0;"
"exit;"
:
: __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0))
: __clobber_all);
}
SEC("socket")
__description("ld_abs: reject in callback")
__failure __msg("cannot use BPF_LD_[ABS|IND] within callback")
int ld_abs_callback_reject(struct __sk_buff *skb)
{
bpf_loop(1, ld_abs_callback, &skb, 0);
return 0;
}
__naked __noinline __used
static int ld_ind_callback_subprog(void)
{
asm volatile (
"r6 = r1;"
"r7 = 0;"
".8byte %[ld_ind];"
"r0 = 0;"
"exit;"
:
: __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0))
: __clobber_all);
}
__naked __noinline __used
static int ld_ind_callback(void)
{
asm volatile (
"r1 = *(u64 *)(r2 + 0);"
"call ld_ind_callback_subprog;"
"exit;"
::: __clobber_all);
}
SEC("socket")
__description("ld_ind: reject in callback subprog")
__failure __msg("cannot use BPF_LD_[ABS|IND] within callback")
int ld_ind_callback_subprog_reject(struct __sk_buff *skb)
{
bpf_loop(1, ld_ind_callback, &skb, 0);
return 0;
}
static __noinline int ld_ind_global_static(struct __sk_buff *skb)
{
asm volatile (
"r6 = %[skb];"
"r7 = 0;"
".8byte %[ld_ind];"
:
: [skb] "r"(skb),
__imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0))
: __clobber_common, "r6", "r7");
return skb->mark;
}
__noinline int ld_ind_global(struct __sk_buff *skb)
{
return ld_ind_global_static(skb);
}
static int ld_ind_global_callback(__u32 index, struct __sk_buff **ctx)
{
ld_ind_global(*ctx);
return 0;
}
SEC("socket")
__description("ld_ind: reject in callback global subprog")
__failure __msg("cannot use BPF_LD_[ABS|IND] within callback")
int ld_ind_global_callback_reject(struct __sk_buff *skb)
{
bpf_loop(1, ld_ind_global_callback, &skb, 0);
return 0;
}
SEC("socket")
__description("ld_ind: allow in non-callback global subprog")
__success
int ld_ind_global_subprog_ok(struct __sk_buff *skb)
{
return ld_ind_global(skb);
}
/*
* ld_{abs,ind} in subprogs require scalar (int) return type in BTF.
* A test with void return must be rejected.

78
tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c
View File

@ -113,4 +113,82 @@ int with_valid_ctx_access_test6(struct bpf_nf_ctx *ctx)
return th->dest == bpf_htons(22) ? NF_ACCEPT : NF_DROP;
}
SEC("netfilter")
__description("netfilter test prog with skb write access")
__failure __msg("only read is supported")
int skb_len_write(struct bpf_nf_ctx *ctx)
{
ctx->skb->len = 1;
return 1;
}
SEC("netfilter")
__description("netfilter test prog with skb data write access")
__failure __msg("cannot write into rdonly_untrusted_mem")
int skb_data_write(struct bpf_nf_ctx *ctx)
{
ctx->skb->data[0] = 0;
return 1;
}
SEC("netfilter")
__description("netfilter test prog with bpf_dynptr_write")
__success __failure_unpriv
__retval(0)
int with_dynptr_write(struct bpf_nf_ctx *ctx)
{
struct __sk_buff *skb = (struct __sk_buff *)ctx->skb;
struct bpf_dynptr ptr;
u8 buffer[1] = {};
if (bpf_dynptr_from_skb(skb, 0, &ptr))
return 1;
if (bpf_dynptr_write(&ptr, 0, buffer, sizeof(buffer), 0))
return 0; /* must always fail */
return 1;
}
SEC("netfilter")
__description("netfilter test prog with bpf_dynptr_slice_rdwr")
__failure __msg("the prog does not allow writes to packet data")
int with_dynptr_rdwr(struct bpf_nf_ctx *ctx)
{
struct __sk_buff *skb = (struct __sk_buff *)ctx->skb;
u8 buffer_iph[20] = {};
struct bpf_dynptr ptr;
struct iphdr *iph;
if (bpf_dynptr_from_skb(skb, 0, &ptr))
return 1;
iph = bpf_dynptr_slice_rdwr(&ptr, 0, buffer_iph, sizeof(buffer_iph));
if (!iph)
return 0;
return 1;
}
SEC("netfilter")
__description("netfilter test prog with bpf_dynptr_slice + write")
__failure __msg("cannot write into rdonly_mem")
int with_dynptr_store(struct bpf_nf_ctx *ctx)
{
struct __sk_buff *skb = (struct __sk_buff *)ctx->skb;
u8 buffer_iph[20] = {};
struct bpf_dynptr ptr;
struct iphdr *iph;
if (bpf_dynptr_from_skb(skb, 0, &ptr))
return 1;
iph = bpf_dynptr_slice(&ptr, 0, buffer_iph, sizeof(buffer_iph));
if (!iph)
return 0;
iph->protocol = 42;
return 1;
}
char _license[] SEC("license") = "GPL";

100
tools/testing/selftests/bpf/progs/verifier_precision.c
View File

@ -2,8 +2,10 @@
/* Copyright (C) 2023 SUSE LLC */
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
#include <stdbool.h>
#include "../../../include/linux/filter.h"
#include "bpf_misc.h"
#include "bpf_kfuncs.h"
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
@ -642,4 +644,102 @@ __naked int bpf_atomic_cmpxchg_32bit_precision(void)
: __clobber_all);
}
/*
* Verification takes two paths: with r1 being scalar zero on path (1)
* and with r1 being some other scalar on path (2).
* Check that the verifier does not use checkpoints created
* on path (1) to prune path (2).
*/
SEC("?tc")
__flag(BPF_F_TEST_STATE_FREQ)
__failure __msg("R1 type=scalar expected=fp")
__naked int null_mem_arg_zero_size(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
"r1 = 42;"
"if r0 > 42 goto 1f;"
"r1 = 0;"
"1:"
"r2 = 0;"
"r3 = 0;"
"r4 = 0;"
"r5 = 0;"
/*
* ARG_PTR_TO_MEM | PTR_MAYBE_NULL parameter can be NULL,
* but can't be some other scalar value.
*/
"call %[bpf_csum_diff];"
"r0 = 0;"
"exit;"
:
: __imm(bpf_get_prandom_u32),
__imm(bpf_csum_diff)
: __clobber_all);
}
__weak int subprog_mem_arg(int *p)
{
if (p)
return *p;
return 0;
}
/*
* Verification takes two paths: with r1 being scalar zero on path (1)
* and with r1 being some other scalar on path (2).
* Check that the verifier does not use checkpoints created
* on path (1) to prune path (2).
*/
SEC("?raw_tp")
__flag(BPF_F_TEST_STATE_FREQ)
__failure __msg("R1 type=scalar expected=fp")
__naked int null_mem_arg_global_subprog(void)
{
asm volatile (
"call %[bpf_get_prandom_u32];"
"r1 = 42;"
"if r0 > 42 goto 1f;"
"r1 = 0;"
"1:"
"call subprog_mem_arg;"
"r0 = 0;"
"exit;"
:
: __imm(bpf_get_prandom_u32)
: __clobber_all);
}
/* Same as above, check that path with r3 == 0 does not prune the path with r3 != 0 */
SEC("?tc")
__flag(BPF_F_TEST_STATE_FREQ)
__failure __msg("R3 type=scalar expected=fp")
int null_kfunc_arg_dynptr_slice(struct __sk_buff *skb)
{
struct bpf_dynptr ptr;
bpf_dynptr_from_skb(skb, 0, &ptr);
asm volatile (
"call %[bpf_get_prandom_u32];"
"r3 = 42;"
"if r0 > 42 goto 1f;"
"r3 = 0;"
"1:"
"r1 = %[ptr];"
"r2 = 0;"
"r4 = 8;"
"call %[bpf_dynptr_slice];"
:
: __imm_ptr(ptr),
__imm(bpf_get_prandom_u32),
__imm(bpf_dynptr_slice)
: __clobber_common);
return 0;
}
void __kfunc_btf_root(void)
{
bpf_dynptr_slice(0, 0, 0, 0);
}
char _license[] SEC("license") = "GPL";

13
tools/testing/selftests/bpf/progs/verifier_scalar_ids.c
View File

@ -380,13 +380,14 @@ SEC("socket")
__success __log_level(2)
__flag(BPF_F_TEST_STATE_FREQ)
/*
* check that r0 and r5 have different IDs after 'if',
* collect_linked_regs() can't tie more than 5 registers for a single insn.
* check that r5 is unlinked after 'if', collect_linked_regs() can't tie
* more than 5 registers for a single insn and the register compared by
* the jump is not exempt from that.
*/
__msg("7: (25) if r0 > 0x7 goto pc+0 ; R0=scalar(id=1")
__msg("7: (25) if r5 > 0x7 goto pc+0 ; R5=scalar(smin=")
__msg("12: (bf) r5 = r5 ; R5=scalar(id=2")
/* check that r{0-4} are marked precise after 'if' */
__msg("frame0: regs=r0 stack= before 7: (25) if r0 > 0x7 goto pc+0")
__msg("frame0: regs=r0 stack= before 7: (25) if r5 > 0x7 goto pc+0")
__msg("frame0: parent state regs=r0,r1,r2,r3,r4 stack=:")
__naked void linked_regs_too_many_regs(void)
{
@ -400,8 +401,8 @@ __naked void linked_regs_too_many_regs(void)
"r3 = r0;"
"r4 = r0;"
"r5 = r0;"
/* propagate range for r{0-5} */
"if r0 > 7 goto +0;"
/* r{0-4} fill the record, r5 does not fit and is unlinked */
"if r5 > 7 goto +0;"
/* keep r{1-4} live */
"r1 = r1;"
"r2 = r2;"

40
tools/testing/selftests/bpf/progs/verifier_spill_fill.c
View File

@ -1403,6 +1403,46 @@ __naked void partial_fill_from_cleaned_pointer_spill(void)
::: __clobber_all);
}
SEC("raw_tp")
__failure
__msg("access may be outside object bounds")
__flag(BPF_F_TEST_STATE_FREQ)
__naked void imprecise_scalar_spill_half_dead(void)
{
asm volatile (
/*
* Fork two paths: the one explored first spills an imprecise zero,
* the one explored second, an imprecise non-zero scalar.
*/
"call %[bpf_get_prandom_u32];"
"if r0 > 42 goto 1f;"
"r6 = 0;"
"goto 2f;"
"1:"
/* causes out of bounds access on a second path. */
"r6 = 100500;"
"2:"
/* Force a checkpoint before the spill. */
"goto +0;"
"*(u64 *)(r10 - 8) = r6;"
/*
* Force stack cleanup, only the low half of the spill is alive,
* so the dead high half is degraded to raw stack bytes.
* Buggy verifier converted it to STACK_ZERO w/o proper precision propagation.
*/
"goto +0;"
"r7 = *(u32 *)(r10 - 4);"
/* Use r7 as an offset into a one-byte buffer. */
"r1 = %[single_byte_buf] ll;"
"r1 += r7;"
"r0 = *(u8 *)(r1 + 0);"
"exit;"
:
: __imm(bpf_get_prandom_u32),
__imm_addr(single_byte_buf)
: __clobber_all);
}
/* check valid spill/fill, ptr to tp buffer */
SEC("raw_tracepoint.w")
__success

63
tools/testing/selftests/bpf/progs/verifier_subprog_precision.c
View File

@ -287,9 +287,9 @@ __msg("17: (b7) r0 = 0")
__msg("18: (95) exit")
__msg("returning from callee:")
__msg("to caller at 9:")
__msg("frame 0: propagating r1,r4")
__msg("frame 0: propagating r1,r3,r4")
__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1")
__msg("mark_precise: frame0: regs=r1,r4 stack= before 18: (95) exit")
__msg("mark_precise: frame0: regs=r1,r3,r4 stack= before 18: (95) exit")
__msg("from 18 to 9: safe")
__naked int callback_result_precise(void)
{
@ -419,9 +419,9 @@ __msg("to caller at 9:")
/* r1, r4 are always precise for bpf_loop(),
* r6 was marked before backtracking to callback body.
*/
__msg("frame 0: propagating r1,r4,r6")
__msg("frame 0: propagating r1,r3,r4,r6")
__msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1")
__msg("mark_precise: frame0: regs=r1,r4,r6 stack= before 16: (95) exit")
__msg("mark_precise: frame0: regs=r1,r3,r4,r6 stack= before 16: (95) exit")
__msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0")
__msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop")
__msg("mark_precise: frame0: parent state regs= stack=:")
@ -575,9 +575,9 @@ __msg("to caller at 10:")
/* r1, r4 are always precise for bpf_loop(),
* fp-8 was marked before backtracking to callback body.
*/
__msg("frame 0: propagating r1,r4,fp-8")
__msg("frame 0: propagating r1,r3,r4,fp-8")
__msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx -1")
__msg("mark_precise: frame0: regs=r1,r4 stack=-8 before 18: (95) exit")
__msg("mark_precise: frame0: regs=r1,r3,r4 stack=-8 before 18: (95) exit")
__msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0")
__msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181")
__msg("mark_precise: frame0: parent state regs= stack=:")
@ -846,4 +846,55 @@ __naked int subprog_result_tail_call(void)
);
}
__naked __noinline __used
static int ld_abs_subprog(void)
{
asm volatile (
"r6 = r1;"
"r7 = r1;"
".8byte %[ld_abs];"
"exit;"
:
: __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0))
: __clobber_all);
}
/*
* Buggy verifier did not properly backtrack early subprogram exit
* modelled for BPF_LD | BPF_ABS instruction, causing a segfault.
*/
SEC("socket")
__success
__log_level(2)
/* early exit path */
__msg("3: (0f) r1 += r7")
__msg("mark_precise: frame0: regs=r7 stack= before 2: (bf) r1 = r10")
__msg("mark_precise: frame0: regs=r7 stack= before 9: (20) r0 = *(u32 *)skb[0]")
__msg("mark_precise: frame1: regs= stack= before 8: (bf) r7 = r1")
__msg("mark_precise: frame1: regs= stack= before 7: (bf) r6 = r1")
__msg("mark_precise: frame1: regs= stack= before 1: (85) call pc+5")
__msg("mark_precise: frame0: regs=r7 stack= before 0: (b7) r7 = -8")
/* fallthrough path */
__msg("3: (0f) r1 += r7")
__msg("mark_precise: frame0: regs=r7 stack= before 2: (bf) r1 = r10")
__msg("mark_precise: frame0: regs=r7 stack= before 10: (95) exit")
__msg("mark_precise: frame1: regs= stack= before 9: (20) r0 = *(u32 *)skb[0]")
__msg("mark_precise: frame1: regs= stack= before 8: (bf) r7 = r1")
__msg("mark_precise: frame1: regs= stack= before 7: (bf) r6 = r1")
__msg("mark_precise: frame1: regs= stack= before 1: (85) call pc+5")
__msg("mark_precise: frame0: regs=r7 stack= before 0: (b7) r7 = -8")
__naked int ld_abs_backtrack_both_paths(void)
{
asm volatile (
"r7 = -8;"
"call ld_abs_subprog;"
"r1 = r10;"
"r1 += r7;" /* mark r7 as precise */
"*(u64 *)(r1 + 0) = 0;"
"r0 = 0;"
"exit;"
::: __clobber_all
);
}
char _license[] SEC("license") = "GPL";

26
tools/testing/selftests/bpf/progs/verifier_zext.c
View File

@ -356,6 +356,32 @@ __naked void arena_ptr(void)
: __clobber_all);
}
/*
* Result of a 32-bit cmpxchg is always explicitly zero extended.
* Check that this holds for arenas (BPF_PROBE_ATOMIC instruction flavor).
*/
SEC("socket")
__success
__xlated("probe r0 = atomic_cmpxchg((u32 *)(r1 +0), r0, r2)")
__xlated("w0 = w0")
__naked void zext_arena_cmpxchg32(void)
{
asm volatile (" \
r9 = %[arena] ll; /* associate the arena with the program */ \
r1 = 0; \
r1 = addr_space_cast(r1, 0, 1); \
r0 = 0; \
r2 = 0; \
.8byte %[cmpxchg32]; \
r0 >>= 32; /* make the upper half live */ \
exit; \
" :
: __imm_addr(arena),
__imm_insn(cmpxchg32,
BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_1, BPF_REG_2, 0))
: __clobber_all);
}
#endif
/* Check if probe mem loads keep their zero extension. */

45
tools/testing/selftests/bpf/verifier/pseudo_func.c Normal file
View File

@ -0,0 +1,45 @@
/*
* Buggy verifier accepted the program below while not patching BPF_PSEUDO_FUNC
* load instruction to contain a real address. Which resulted in a function call
* to a bogus address.
*/
{
"BPF_PSEUDO_FUNC reference to the main program",
.insns = {
/* r6 = bpf_map_lookup_elem(&timer_map, &(int){0}); */
BPF_ST_MEM(BPF_W, BPF_REG_10, -4, 0),
BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
BPF_LD_MAP_FD(BPF_REG_1, 0),
BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10),
BPF_MOV64_REG(BPF_REG_6, BPF_REG_0),
/* bpf_timer_init(r6, &timer_map, 0); */
BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
BPF_LD_MAP_FD(BPF_REG_2, 0),
BPF_MOV64_IMM(BPF_REG_3, 0),
BPF_EMIT_CALL(BPF_FUNC_timer_init),
/* bpf_timer_set_callback(r6, <insn #0>); */
BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, -15),
BPF_RAW_INSN(0, 0, 0, 0, 0),
BPF_EMIT_CALL(BPF_FUNC_timer_set_callback),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.prog_type = BPF_PROG_TYPE_TRACEPOINT,
.fixup_map_timer = { 3, 9 },
.result = REJECT,
.errstr = "callback function cannot be the main program",
.func_info = { { 0, 4 /* main_prog */ } },
.func_info_cnt = 1,
.btf_strings = "\0int\0ctx\0main_prog",
.btf_types = {
/* 1: int */ BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
/* 2: void* */ BTF_PTR_ENC(0),
/* 3: int __(void *) */ BTF_FUNC_PROTO_ENC(1, 1),
BTF_FUNC_PROTO_ARG_ENC(5, 2),
/* 4: main_prog */ BTF_FUNC_ENC(9, 3),
BTF_END_RAW
}
},