diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 73578a4c3b1f..901c37504217 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -407,6 +407,13 @@ referenced kptr (by invoking bpf_kptr_xchg). If not, the verifier fails the loading of the BPF program until no lingering references remain in all possible explored states of the program. +A pointer to a struct in the arguments of a KF_ACQUIRE kfunc accepts only an +object of that type. The same applies to the argument that a KF_RELEASE kfunc +releases. For other arguments a kfunc also accepts memory of the program, if the +struct has only scalar members. A struct without members, struct __sk_buff and +struct xdp_md never accept memory. A KF_ACQUIRE kfunc which wants such memory +has to pair the pointer with a __sz argument. + 2.5.2 KF_RET_NULL flag ---------------------- diff --git a/arch/arm64/net/bpf_timed_may_goto.S b/arch/arm64/net/bpf_timed_may_goto.S index 1b4a4f91ffc9..f8732b13bb1e 100644 --- a/arch/arm64/net/bpf_timed_may_goto.S +++ b/arch/arm64/net/bpf_timed_may_goto.S @@ -17,16 +17,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) str x7, [sp, #80] /* - * Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP - * (x25) to get the pointer to count and timestamp and pass it as the - * first argument in x0. - * - * Before generating the call to arch_bpf_timed_may_goto, the verifier - * generates a load instruction using FP, i.e. REG_AX = *(u64 *)(FP - - * stack_off_cnt), so BPF_REG_FP (x25) is always set up by the arm64 - * jit in this case. + * The pointer to count and timestamp was passed in BPF_REG_AX (x9), + * pass it as the first argument in x0. */ - add x0, x9, x25 + mov x0, x9 bl bpf_check_timed_may_goto /* BPF_REG_AX(x9) will be stored into count, so move return value to it. */ mov x9, x0 diff --git a/arch/loongarch/net/bpf_timed_may_goto.S b/arch/loongarch/net/bpf_timed_may_goto.S index fa128acdb108..4aaca029404b 100644 --- a/arch/loongarch/net/bpf_timed_may_goto.S +++ b/arch/loongarch/net/bpf_timed_may_goto.S @@ -22,14 +22,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) st.d a4, sp, 48 /* - * BPF_REG_AX (t0) holds the offset passed in by the verifier; - * add it to BPF_REG_FP (s4) to get the pointer to the count and - * timestamp, then pass it as the first argument in a0. - * - * The verifier emits a load using FP right before this call, - * so BPF_REG_FP (s4) is always set up by the JIT in this case. + * BPF_REG_AX (t0) holds the pointer to the count and timestamp; + * pass it as the first argument in a0. */ - add.d a0, t0, s4 + move a0, t0 bl bpf_check_timed_may_goto /* BPF_REG_AX (t0) will be stored into count, so move the return value to it. */ move t0, a0 diff --git a/arch/powerpc/net/bpf_timed_may_goto.S b/arch/powerpc/net/bpf_timed_may_goto.S index 84ecf6fa7f5d..b8ae9885eb31 100644 --- a/arch/powerpc/net/bpf_timed_may_goto.S +++ b/arch/powerpc/net/bpf_timed_may_goto.S @@ -8,7 +8,7 @@ * arch_bpf_timed_may_goto() trampoline for powerpc64 * * Custom BPF convention (verifier/JIT): - * - input: stack offset in BPF_REG_AX (r12) + * - input: pointer to count and timestamp in BPF_REG_AX (r12) * - output: updated count in BPF_REG_AX (r12) * * Call bpf_check_timed_may_goto(ptr) with normal powerpc64 ABI: @@ -31,11 +31,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) std r7, 64(r1) std r8, 72(r1) - /* - * r3 = BPF_REG_FP + BPF_REG_AX - * BPF_REG_FP is r31; BPF_REG_AX is r12 (stack offset in bytes). - */ - add r3, r31, r12 + /* r3 = BPF_REG_AX (r12) */ + mr r3, r12 bl CFUNC(bpf_check_timed_may_goto) /* Put return value back into AX */ diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index 6ec37f634aff..2ce1c89c913e 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -6,13 +6,11 @@ /* * Trampoline for the BPF timed may_goto loop bound. Custom calling convention: - * - input: stack offset in BPF_REG_AX (t0) + * - input: pointer to count and timestamp in BPF_REG_AX (t0) * - output: updated count in BPF_REG_AX (t0) * - * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) and stack - * arguments 6-8 (a5-a7) are saved across the call; BPF_REG_FP (s5) is - * callee-saved and needs no saving. + * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI. BPF R0-R5 + * (t6, a0-a4) and stack arguments 6-8 (a5-a7) are saved across the call. */ SYM_FUNC_START(arch_bpf_timed_may_goto) @@ -33,7 +31,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S a6, 7*SZREG(sp) REG_S a7, 8*SZREG(sp) - add a0, t0, s5 + mv a0, t0 call bpf_check_timed_may_goto mv t0, a0 diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 56ec49894dda..3ab880fc17a6 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1005,6 +1005,17 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, (jit->prg - patch_brc) >> 1; break; } + if (insn_is_mov_percpu_addr(insn)) { + const u64 *percpu_offset = &get_lowcore()->percpu_offset; + + if (dst_reg != src_reg) + /* lgr %dst,%src */ + EMIT4(0xb9040000, dst_reg, src_reg); + /* ag %dst,percpu_offset */ + EMIT6_DISP_LH(0xe3000000, 0x0008, dst_reg, REG_0, REG_0, + (unsigned long)percpu_offset); + break; + } switch (insn->off) { case 0: /* DST = SRC */ /* lgr %dst,%src */ @@ -1866,9 +1877,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if ((void *)func == arch_bpf_timed_may_goto) { /* * arch_bpf_timed_may_goto() has a special ABI: the - * parameters are in BPF_REG_AX and BPF_REG_10; the - * return value is in BPF_REG_AX; and all GPRs except - * REG_W0, REG_W1, and BPF_REG_AX are callee-saved. + * parameter is in BPF_REG_AX; the return value is in + * BPF_REG_AX; and all GPRs except REG_W0, REG_W1, and + * BPF_REG_AX are callee-saved. */ /* brasl %r0,func */ @@ -2428,8 +2439,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr fp->jited_len = jit.size - cfi_get_offset(); if (!fp->is_func || extra_pass) { - for (int i = 0; i < fp->len; i++) - jit.addrs[i] -= cfi_get_offset(); bpf_prog_fill_jited_linfo(fp, jit.addrs + 1); free_addrs: kvfree(jit.addrs); @@ -3116,6 +3125,11 @@ bool bpf_jit_supports_timed_may_goto(void) return true; } +bool bpf_jit_supports_percpu_insn(void) +{ + return true; +} + bool bpf_jit_inlines_helper_call(s32 imm) { switch (imm) { diff --git a/arch/s390/net/bpf_timed_may_goto.S b/arch/s390/net/bpf_timed_may_goto.S index 06f567a460d7..05f532f8c639 100644 --- a/arch/s390/net/bpf_timed_may_goto.S +++ b/arch/s390/net/bpf_timed_may_goto.S @@ -21,9 +21,9 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* - * This function has a special ABI: the parameters are in %r12 and - * %r13; the return value is in %r12; all GPRs except %r0, %r1, and - * %r12 are callee-saved; and the return address is in %r0. + * This function has a special ABI: the parameter is in %r12; the + * return value is in %r12; all GPRs except %r0, %r1, and %r12 are + * callee-saved; and the return address is in %r0. */ stmg %r2,%r5,FRAME_OFF+R2_OFF(%r15) stg %r14,FRAME_OFF+R14_OFF(%r15) @@ -33,7 +33,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) lay %r15,-FRAME_SIZE(%r15) stg %r1,__SF_BACKCHAIN(%r15) - lay %r2,0(%r12,%r13) + lgr %r2,%r12 brasl %r14,bpf_check_timed_may_goto lgr %r12,%r2 diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 62c1c93aa1c6..4b20673a3395 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -286,9 +286,9 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) RET 1: - testb $1, %al - jz 2f - andq $0xfffffffffffffffe, %rax + /* See ftrace_jmp_set(), jmp to the trampoline */ + btsq $63, %rax + jc 2f movq %rax, MCOUNT_REG_SIZE+8(%rsp) restore_mcount_regs /* Restore flags */ diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index d90b6d238b3a..4c3e09d98131 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2626,8 +2626,13 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * * Arg 6 goes into r9 register, not on stack. */ outgoing_rsp = out_stack_arg_cnt > 1 ? (out_stack_arg_cnt - 1) * 8 : 0; + /* + * bpf_throw() sees rsp of this frame as it is at a call. The callee-saved + * regs are above the outgoing args, or above r9, which is pushed around + * every call when the stack is private. + */ if (bpf_prog->aux->exception_boundary) - bpf_prog->aux->stack_arg_sp_adjust = outgoing_rsp; + bpf_prog->aux->stack_arg_sp_adjust = outgoing_rsp + (priv_frame_ptr ? 8 : 0); emit_sub_rsp(&prog, outgoing_rsp); if (arena_vm_start) @@ -3436,18 +3441,6 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * ip += emit_kfunc_arg_moves(fm, outgoing_arg_base - outgoing_rsp, &prog); } - if (func == (u8 *)arch_bpf_timed_may_goto) { - u32 fp = priv_frame_ptr ? X86_REG_R9 : BPF_REG_FP; - - /* - * AX has the offset of count and timestamp - * in the stack. Turn it into a pointer. - * add r10, rbp or add r10, r9 - */ - maybe_emit_mod(&prog, BPF_REG_AX, fp, true); - EMIT2(0x01, add_2reg(0xC0, BPF_REG_AX, fp)); - ip += 3; - } if (priv_frame_ptr) { push_r9(&prog); ip += 2; diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index dd2dde89836b..da42de7cca98 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -25,10 +25,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* Save stack argument 6. */ pushq %r9 - /* - * r10 has the pointer to count and timestamp. JIT added the frame - * pointer to the offset. Pass it as first argument. - */ + /* r10 has the pointer to count and timestamp. Pass it as first argument. */ movq %r10, %rdi /* Emit call depth accounting for call below. */ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ff0436781293..fdbf638300fc 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1752,12 +1752,22 @@ enum bpf_func_context { BPF_FUNC_CTX_CNT, }; +struct bpf_subprog_arg_info { + enum bpf_arg_type arg_type; + union { + u32 mem_size; + u32 btf_id; + }; +}; + struct bpf_func_info_aux { u16 linkage; bool unreliable; bool called[BPF_FUNC_CTX_CNT]; bool verified[BPF_FUNC_CTX_CNT]; bool called_with_pkt_ptrs; + /* what the callers of a global function are checked against */ + struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS]; }; enum bpf_jit_poke_reason { @@ -1916,6 +1926,8 @@ struct bpf_prog_aux { const char *attach_func_name; struct bpf_prog **func; struct bpf_prog_aux *main_prog_aux; + /* main prog: there when the prog has timed may_goto */ + struct bpf_timed_may_goto __percpu *timed_may_goto; void *jit_data; /* JIT specific data. arch dependent */ struct bpf_jit_poke_descriptor *poke_tab; struct bpf_kfunc_desc_tab *kfunc_tab; @@ -3044,6 +3056,8 @@ static inline void bpf_map_dec_elem_count(struct bpf_map *map) this_cpu_dec(*map->elem_count); } +s64 bpf_map_sum_elem_count(const struct bpf_map *map); + extern int sysctl_unprivileged_bpf_disabled; bool bpf_token_capable(const struct bpf_token *token, int cap); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 7baeb831281f..b43eb97bcd44 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -50,6 +50,13 @@ enum bpf_iter_state { BPF_ITER_STATE_DRAINED, }; +/* Bytes [off, off + size) of the stack of frame frameno. Nothing when size is 0. */ +struct bpf_stack_range { + u32 frameno; + s32 off; + u32 size; +}; + struct bpf_reg_state { /* Ordering of fields matters. See states_equal() */ enum bpf_reg_type type; @@ -83,6 +90,12 @@ struct bpf_reg_state { struct { /* for PTR_TO_MEM | PTR_TO_MEM_OR_NULL */ u32 mem_size; + /* + * bpf_dynptr_slice() and bpf_dynptr_slice_rdwr() return the + * buffer when the data is not contiguous: the register may + * point to this part of the stack. + */ + struct bpf_stack_range buf; }; /* For dynptr stack slots */ @@ -923,14 +936,6 @@ struct bpf_func_ptr { /* the subprog that a bpf_func_ptr pointed to was removed as dead code */ #define BPF_FUNC_PTR_DELETED ((u32)-1) -struct bpf_subprog_arg_info { - enum bpf_arg_type arg_type; - union { - u32 mem_size; - u32 btf_id; - }; -}; - enum priv_stack_mode { PRIV_STACK_UNKNOWN, NO_PRIV_STACK, @@ -964,6 +969,10 @@ struct bpf_subprog_info { bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; + /* bpf_loop() or bpf_for_each_map_elem() call it in a loop */ + bool is_loop_cb: 1; + /* BPF code, another helper or a kfunc call it */ + bool is_called: 1; bool args_cached: 1; /* true if the return value is passed in the R0:R2 register pair */ bool ret_reg_pair: 1; @@ -1099,6 +1108,8 @@ struct bpf_verifier_env { bool bypass_spec_v1; bool bypass_spec_v4; bool seen_direct_write; + /* a register may point to the stack and is not PTR_TO_STACK, see bpf_reg_state::buf */ + bool seen_stack_slice; /* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */ bool no_rcu_read_lock; bool seen_exception; @@ -1823,6 +1834,8 @@ struct bpf_call_arg_meta { struct arg_constant_desc arg_constant; /* The type of the dynptr that the kfunc writes through */ enum bpf_dynptr_type dynptr_write_type; + /* The buffer of bpf_dynptr_slice() or bpf_dynptr_slice_rdwr() */ + struct bpf_stack_range slice_buf; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later @@ -1900,6 +1913,7 @@ bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, const struct bpf_reg_state *reg_b); +void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax); int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, struct cnum64 r64, struct cnum32 r32, u16 base, u16 step); bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b); diff --git a/include/linux/btf.h b/include/linux/btf.h index a18254b7521b..87fad4076f0e 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -631,6 +631,7 @@ s32 btf_find_dtor_kfunc(struct btf *btf, u32 btf_id); int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt, struct module *owner); struct btf_struct_meta *btf_find_struct_meta(const struct btf *btf, u32 btf_id); +bool btf_is_projection_type(const char *pname); bool btf_is_projection_of(const char *pname, const char *tname); bool btf_is_prog_ctx_type(struct bpf_verifier_log *log, const struct btf *btf, const struct btf_type *t, enum bpf_prog_type prog_type, diff --git a/include/linux/filter.h b/include/linux/filter.h index 98254742ae37..4f0cd67caccb 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -758,9 +758,15 @@ struct bpf_prog_stats { struct u64_stats_sync syncp; } __aligned(2 * sizeof(u64)); +/* + * Per-cpu memory of a prog. All may_goto of one invocation of the prog use it. An invocation is + * one call from the kernel into the main prog, with the subprogs and sync callbacks it reaches. + */ struct bpf_timed_may_goto { u64 count; u64 timestamp; + /* the async callback that has set count and timestamp, see bpf_timed_may_goto_save() */ + void *owner; }; struct sk_filter { @@ -1328,6 +1334,8 @@ u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *); +void bpf_timed_may_goto_save(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved); +void bpf_timed_may_goto_restore(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved); bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id); static inline bool bpf_dump_raw_ok(const struct cred *cred) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index b041d9142d2b..1c2056a1a68b 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -618,19 +618,21 @@ static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, #endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ #ifdef CONFIG_DYNAMIC_FTRACE_WITH_JMP +#define FTRACE_JMP_BIT BIT(BITS_PER_LONG - 1) + static inline bool ftrace_is_jmp(unsigned long addr) { - return addr & 1; + return !(addr & FTRACE_JMP_BIT); } static inline unsigned long ftrace_jmp_set(unsigned long addr) { - return addr | 1UL; + return addr & ~FTRACE_JMP_BIT; } static inline unsigned long ftrace_jmp_get(unsigned long addr) { - return addr & ~1UL; + return addr | FTRACE_JMP_BIT; } #else static inline bool ftrace_is_jmp(unsigned long addr) diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index 3c61c181a6fa..ec2bda1a6ebf 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -7,6 +7,7 @@ #include #include "linux/filter.h" #include +#include #include #include #include @@ -630,11 +631,11 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad { struct bpf_map *map = filp->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); - long ret; + unsigned long ret; if (pgoff) return -EINVAL; - if (len > SZ_4G) + if (len > (u64)map->max_entries * PAGE_SIZE) return -E2BIG; /* if user_vm_start was specified at arena creation time */ @@ -647,14 +648,22 @@ static unsigned long arena_get_unmapped_area(struct file *filp, unsigned long ad return -EINVAL; } - ret = mm_get_unmapped_area(filp, addr, len * 2, 0, flags); + /* + * Without MAP_FIXED the mm finds a free range. Ask for twice the + * length, so that there is room for round_up(). With MAP_FIXED the mm + * does no search, and the caller wants this address. Thus the range + * cannot move. + */ + ret = mm_get_unmapped_area(filp, addr, flags & MAP_FIXED ? len : len * 2, 0, flags); if (IS_ERR_VALUE(ret)) return ret; + if (!ret) + /* user_vm_start == 0 means that the user range is not set yet */ + return -EINVAL; if ((ret >> 32) == ((ret + len - 1) >> 32)) return ret; - if (WARN_ON_ONCE(arena->user_vm_start)) - /* checks at map creation time should prevent this */ - return -EFAULT; + if (flags & MAP_FIXED) + return -EINVAL; return round_up(ret, SZ_4G); } @@ -679,7 +688,7 @@ static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) return -EBUSY; /* Earlier checks should prevent this */ - if (WARN_ON_ONCE(vma->vm_end - vma->vm_start > SZ_4G || vma->vm_pgoff)) + if (WARN_ON_ONCE(vma_pages(vma) > map->max_entries || vma->vm_pgoff)) return -EFAULT; if (remember_vma(arena, vma)) @@ -988,6 +997,8 @@ static int arena_free_range(struct bpf_arena *arena, long uaddr, long page_cnt) static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, bool sleepable) { + /* The first mmap() may be setting the user range right now */ + u64 user_vm_start = READ_ONCE(arena->user_vm_start); struct mem_cgroup *new_memcg, *old_memcg; u64 full_uaddr, uaddr_end; struct arena_free_span *s; @@ -995,8 +1006,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, /* only aligned lower 32-bit are relevant */ uaddr = (u32)uaddr; uaddr &= PAGE_MASK; - full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; - if (full_uaddr < arena->user_vm_start) + full_uaddr = clear_lo32(user_vm_start) + uaddr; + if (!user_vm_start || full_uaddr < user_vm_start) return; uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); if (full_uaddr >= uaddr_end) diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index 43d5740034c0..2d6535f271a6 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -746,6 +746,17 @@ void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, i, -(j + 1) * 8); } } + for (j = 0; j < func->out_stack_arg_cnt; j++) { + reg = &func->stack_arg_regs[j]; + if (reg->type != SCALAR_VALUE || reg->precise) + continue; + reg->precise = true; + if (env->log.level & BPF_LOG_LEVEL2) { + verbose(env, + "force_precise: frame%d: forcing r11%d to be precise\n", + i, -(j + 1) * BPF_REG_SIZE); + } + } } } } diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 1be30487a2fa..8bf39098ba97 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6590,6 +6590,15 @@ static int find_kern_ctx_type_id(enum bpf_prog_type prog_type) return ctx_type->type; } +/* + * A projection type is the view that programs have of a kernel object with + * another layout: see btf_is_projection_of() for the two types behind them. + */ +bool btf_is_projection_type(const char *pname) +{ + return strcmp(pname, "__sk_buff") == 0 || strcmp(pname, "xdp_md") == 0; +} + bool btf_is_projection_of(const char *pname, const char *tname) { if (strcmp(pname, "__sk_buff") == 0 && strcmp(tname, "sk_buff") == 0) @@ -8483,9 +8492,8 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, } /* This is an optional check to make program writing easier. * Compare names of structs and report an error to the user. - * btf_prepare_func_args() already checked that t2 struct - * is a context type. btf_prepare_func_args() will check - * later that t1 struct is a context type as well. + * check_ext_args() compares what btf_prepare_func_args() + * derives for the two arguments. */ s1 = btf_name_by_offset(btf1, t1->name_off); s2 = btf_name_by_offset(btf2, t2->name_off); @@ -9019,6 +9027,11 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return err; sub->arg_slot_cnt = slots_used; + /* check_ext_args() needs them after env is freed */ + if (is_global) + memcpy(subprog_aux(env, subprog)->args, sub->args, + sizeof(subprog_aux(env, subprog)->args)); + sub->args_cached = true; return 0; diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index ce82963bcfc7..57a785241d42 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -1279,9 +1279,14 @@ static struct bpf_prog_list *find_detach_entry(struct hlist_head *progs, return ERR_PTR(-ENOENT); /* to maintain backward compatibility NONE and OVERRIDE cgroups - * allow detaching with invalid FD (prog==NULL) in legacy mode + * allow detaching with invalid FD (prog==NULL) in legacy mode. + * Without the FD of the attached program the caller needs + * CAP_NET_ADMIN or CAP_SYS_ADMIN in the initial user namespace. */ - return hlist_entry(progs->first, typeof(*pl), node); + pl = hlist_entry(progs->first, typeof(*pl), node); + if (pl->prog != prog && !bpf_net_capable()) + return ERR_PTR(-EPERM); + return pl; } if (!prog && !link) @@ -1785,7 +1790,7 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk, struct sk_buff *skb, enum cgroup_bpf_attach_type atype) { - unsigned int offset = -skb_network_offset(skb); + int offset = skb_network_offset(skb); struct sock *save_sk; void *saved_data_end; struct cgroup *cgrp; @@ -1797,7 +1802,15 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk, cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data); save_sk = skb->sk; skb->sk = sk; - __skb_push(skb, offset); + + /* + * The program sees the packet from its network header on, on + * whichever side of skb->data that header lies. + */ + if (offset > 0) + __skb_pull(skb, offset); + else + __skb_push(skb, -offset); /* compute pointers for the bpf prog */ bpf_compute_and_save_data_end(skb, &saved_data_end); @@ -1839,7 +1852,10 @@ int __cgroup_bpf_run_filter_skb(struct sock *sk, ret = -EFAULT; } bpf_restore_data_end(skb, saved_data_end); - __skb_pull(skb, offset); + if (offset > 0) + __skb_push(skb, offset); + else + __skb_pull(skb, -offset); skb->sk = save_sk; return ret; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7cf37690d41f..831f73f82599 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -206,7 +206,9 @@ void bpf_prog_jit_attempt_done(struct bpf_prog *prog) * here is relative to the prog itself instead of the main prog. * This array has one entry for each xlated bpf insn. * - * jited_off is the byte off to the end of the jited insn. + * jited_off is the byte off to the end of the jited insn, counted from + * the start of the jited image, which is cfi_get_offset() bytes before + * prog->bpf_func. * * Hence, with * insn_start: @@ -220,7 +222,7 @@ void bpf_prog_jit_attempt_done(struct bpf_prog *prog) * * For i > linfo_idx, * - * jited_linfo[i] = prog->bpf_func + + * jited_linfo[i] = prog->bpf_func - cfi_get_offset() + * insn_to_jit_off[linfo[i].insn_off - insn_start - 1] */ void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, @@ -248,7 +250,7 @@ void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, /* The verifier ensures that linfo[i].insn_off is * strictly increasing */ - jited_linfo[i] = prog->bpf_func + + jited_linfo[i] = prog->bpf_func - cfi_get_offset() + insn_to_jit_off[linfo[i].insn_off - insn_start - 1]; } @@ -292,6 +294,7 @@ void __bpf_prog_free(struct bpf_prog *fp) mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); kfree(fp->aux->poke_tab); + free_percpu(fp->aux->timed_may_goto); kfree(fp->aux); } free_percpu(fp->stats); @@ -3643,6 +3646,38 @@ u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p) return BPF_MAX_TIMED_LOOPS; } +/* + * The main prog sets count and timestamp in its first insns. A call of an async callback is an + * invocation of its own too, but it can interrupt an invocation of the prog on this cpu, or go on + * next to one that is preempted. So its caller sets count and timestamp, and puts the old ones + * back at the end. Not when another invocation has set them in the meantime: they are for that + * one now. + * The caller has disabled migration. + */ +void bpf_timed_may_goto_save(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved) +{ + struct bpf_timed_may_goto *cur; + + if (!prog->aux->timed_may_goto) + return; + cur = this_cpu_ptr(prog->aux->timed_may_goto); + *saved = *cur; + cur->count = BPF_MAX_TIMED_LOOPS; + cur->timestamp = 0; + cur->owner = saved; +} + +void bpf_timed_may_goto_restore(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved) +{ + struct bpf_timed_may_goto *cur; + + if (!prog->aux->timed_may_goto) + return; + cur = this_cpu_ptr(prog->aux->timed_may_goto); + if (cur->owner == saved) + *cur = *saved; +} + /* for configs without MMU or 32-bit */ __weak const struct bpf_map_ops arena_map_ops; __weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena) diff --git a/kernel/bpf/diagnostics.c b/kernel/bpf/diagnostics.c index b5204798231d..08e12714bb95 100644 --- a/kernel/bpf/diagnostics.c +++ b/kernel/bpf/diagnostics.c @@ -1561,7 +1561,8 @@ static void diag_record_mod(struct bpf_verifier_env *env, u32 insn_idx, if (origin) { event.mod.origin = *origin; event.mod.origin_valid = true; - } else if (diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { + } else if (reason != BPF_DIAG_MOD_SLICE_BUF && /* the insn writes the buffer */ + diag_mod_insn_origin(env, insn_idx, &target, &event.mod.origin)) { event.mod.origin_valid = true; } if (old_reg && new_reg && reason == BPF_DIAG_MOD_WRITE && @@ -2237,6 +2238,9 @@ static void diag_print_mod(struct bpf_verifier_env *env, const struct bpf_diag_h case BPF_DIAG_MOD_PKT_DATA_CHANGE: reason = "packet data may have moved"; break; + case BPF_DIAG_MOD_SLICE_BUF: + reason = "the stack buffer of this dynptr slice was written, or its frame returned"; + break; case BPF_DIAG_MOD_NON_OWN_REF: reason = "leaving the protected region invalidated this borrowed pointer"; break; diff --git a/kernel/bpf/diagnostics.h b/kernel/bpf/diagnostics.h index a4102fb049ec..490decc133f6 100644 --- a/kernel/bpf/diagnostics.h +++ b/kernel/bpf/diagnostics.h @@ -23,6 +23,7 @@ enum bpf_diag_mod_reason { BPF_DIAG_MOD_VAR_WRITE, BPF_DIAG_MOD_REF_RELEASE, BPF_DIAG_MOD_PKT_DATA_CHANGE, + BPF_DIAG_MOD_SLICE_BUF, BPF_DIAG_MOD_NON_OWN_REF, BPF_DIAG_MOD_CALLER_SAVED, }; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 41bc37e205e0..a656634fa9e4 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1803,6 +1803,50 @@ static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, return cnt + tail_cnt; } +/* + * Timed may_goto. All may_goto of a prog use one count and one timestamp: per-cpu memory of the + * prog, set in the first insns of the main prog, and by the callers of async callbacks. + * @reg holds the address. It is a register that is dead at the may_goto, or R0 that is saved at + * @save_off in the stack. + */ +static int may_goto_expand_timed(struct bpf_insn *insn_buf, int off, + struct bpf_timed_may_goto __percpu *tmg, int reg, int save_off) +{ + struct bpf_insn ld_tmg[] = { BPF_LD_IMM64(reg, (__force unsigned long)tmg) }; + int cnt = 0, jmp, tail_cnt = save_off ? 6 : 5; + + if (save_off) + insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, reg, save_off); + insn_buf[cnt++] = ld_tmg[0]; + insn_buf[cnt++] = ld_tmg[1]; + insn_buf[cnt++] = BPF_MOV64_PERCPU_REG(reg, reg); + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, reg, 0); + if (save_off) { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_10, save_off); + jmp = cnt++; + insn_buf[jmp] = BPF_JMP32_A(off >= 0 ? off + tail_cnt : off - jmp); + } else { + jmp = cnt; + off = off >= 0 ? off + tail_cnt : off - jmp; + if (off == (s16)off) { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); + } else { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); + insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); + } + } + insn_buf[cnt++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); + /* AX is the argument and the return value */ + insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_AX, reg); + insn_buf[cnt++] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); + insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, reg, BPF_REG_AX, 0); + if (save_off) + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_10, save_off); + return cnt; +} + /* Do various post-verification rewrites in a single program pass. * These rewrites simplify JIT and interpreter implementations. */ @@ -1823,6 +1867,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) struct bpf_subprog_info *subprogs = env->subprog_info; u16 stack_depth = subprogs[cur_subprog].stack_depth; u16 stack_depth_extra = 0; + /* arch_bpf_timed_may_goto() takes and returns AX in a cpu register: for JITs only */ + bool timed_may_goto = prog->jit_requested && bpf_jit_supports_timed_may_goto(); + struct bpf_timed_may_goto __percpu *tmg = NULL; if (env->seen_exception && !env->exception_callback_subprog) { struct bpf_insn *patch = insn_buf; @@ -2078,38 +2125,28 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } - if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { - int stack_off_cnt = -stack_depth - 16; - /* - * AX is used as an argument to pass in stack_off_cnt - * (to add to r10/fp), and also as the return value of - * the call to arch_bpf_timed_may_goto. - */ - struct bpf_insn tail[] = { - BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), - BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), - BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), - BPF_EMIT_CALL(arch_bpf_timed_may_goto), - BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), - }; + if (bpf_is_may_goto_insn(insn) && timed_may_goto) { + u16 live = env->insn_aux_data[i + delta].live_regs_before; + int reg, save_off = 0; - /* - * Two 8 byte slots, depth-16 stores the count, and - * depth-8 stores the start timestamp of the loop. - * - * The starting value of count is BPF_MAX_TIMED_LOOPS - * (0xffff). Every iteration loads it and subs it by 1, - * until the value becomes 0 in AX (thus, 1 in stack), - * after which we call arch_bpf_timed_may_goto, which - * either sets AX to 0xffff to keep looping, or to 0 - * upon timeout. AX is then stored into the stack. In - * the next iteration, we either see 0 and break out, or - * continue iterating until the next time value is 0 - * after subtraction, rinse and repeat. - */ - stack_depth_extra = 16; - cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, - tail, ARRAY_SIZE(tail)); + /* a register that is dead here holds the address of the count */ + for (reg = BPF_REG_0; reg <= BPF_REG_9; reg++) + if (!(live & BIT(reg))) + break; + if (reg > BPF_REG_9) { + reg = BPF_REG_0; + save_off = -stack_depth - 8; + stack_depth_extra = 8; + } + if (!tmg) { + tmg = alloc_percpu_gfp(struct bpf_timed_may_goto, + GFP_KERNEL_ACCOUNT); + if (!tmg) + return -ENOMEM; + prog->aux->timed_may_goto = tmg; + } + prog->jit_required = 1; + cnt = may_goto_expand_timed(insn_buf, insn->off, tmg, reg, save_off); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2730,24 +2767,52 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) env->prog->aux->stack_depth = subprogs[0].stack_depth; for (i = 0; i < env->subprog_cnt; i++) { - int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; int subprog_start = subprogs[i].start; int stack_slots = subprogs[i].stack_extra / 8; - int slots = delta, cnt = 0; + /* an invocation begins, see bpf_timed_may_goto_save() for async callbacks */ + bool main_prog = tmg && !i; + /* only a helper calls it, in a loop that ends when it returns 1 */ + bool loop_cb = tmg && i && subprogs[i].is_loop_cb && !subprogs[i].is_called && + !subprogs[i].is_async_cb; + int cnt = 0; - if (!stack_slots) + if (timed_may_goto ? !main_prog && !loop_cb : !stack_slots) continue; - /* We need two slots in case timed may_goto is supported. */ - if (stack_slots > slots) { + if (stack_slots > 1) { verifier_bug(env, "stack_slots supports may_goto only"); return -EFAULT; } stack_depth = subprogs[i].stack_depth; - if (bpf_jit_supports_timed_may_goto()) { - insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, - BPF_MAX_TIMED_LOOPS); - insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); + if (timed_may_goto) { + struct bpf_insn ld_tmg[] = { + BPF_LD_IMM64(BPF_REG_0, (__force unsigned long)tmg) + }; + int count_off = offsetof(struct bpf_timed_may_goto, count); + int timestamp_off = offsetof(struct bpf_timed_may_goto, timestamp); + int owner_off = offsetof(struct bpf_timed_may_goto, owner); + + /* R0 has nothing at the entry */ + insn_buf[cnt++] = ld_tmg[0]; + insn_buf[cnt++] = ld_tmg[1]; + insn_buf[cnt++] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); + if (main_prog) { + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, count_off, + BPF_MAX_TIMED_LOOPS); + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, timestamp_off, 0); + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, owner_off, 0); + } else { + /* + * The time is up, but the helper would call the function again and + * again. End its loop. The verifier has seen that: the loop can end + * after each call. + */ + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, + count_off); + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2); + insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 1); + insn_buf[cnt++] = BPF_EXIT_INSN(); + } } else { /* Add ST insn to subprog prologue to init extra stack */ insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, @@ -2774,7 +2839,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * to insn after BPF_ST that inits may_goto count. * Adjustment will succeed because bpf_patch_insn_data() didn't fail. */ - WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); + WARN_ON(adjust_jmp_off(env->prog, subprog_start, cnt - 1)); } /* Since poke tab is now finalized, publish aux to tracker. */ diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index bf78e09f4d51..6e2115477cb9 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -94,10 +94,10 @@ struct bpf_htab { struct bpf_lru lru; }; struct htab_elem *__percpu *extra_elems; - /* number of elements in non-preallocated hashtable are kept - * in either pcount or count + /* number of elements in non-preallocated hashtable. With + * use_percpu_counter it only follows map.elem_count in steps of + * PERCPU_COUNTER_BATCH, see is_map_full() */ - struct percpu_counter pcount; atomic_t count; bool use_percpu_counter; bool has_hash; @@ -739,29 +739,20 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) htab_init_buckets(htab); -/* compute_batch_value() computes batch value as num_online_cpus() * 2 - * and __percpu_counter_compare() needs - * htab->max_entries - cur_number_of_elems to be more than batch * num_online_cpus() - * for percpu_counter to be faster than atomic_t. In practice the average bpf - * hash map size is 10k, which means that a system with 64 cpus will fill - * hashmap to 20% of 10k before percpu_counter becomes ineffective. Therefore - * define our own batch count as 32 then 10k hash map can be filled up to 80%: +/* is_map_full() needs htab->max_entries - cur_number_of_elems to be more than + * batch * nr_cpu_ids for the percpu counter to be faster than atomic_t. In + * practice the average bpf hash map size is 10k. With batch count of 32 a 10k + * hash map on a system with 64 cpus can be filled up to 80%: * 10k - 8k > 32 _batch_ * 64 _cpus_ - * and __percpu_counter_compare() will still be fast. At that point hash map + * and is_map_full() will still be fast. At that point hash map * collisions will dominate its performance anyway. Assume that hash map filled * to 50+% isn't going to be O(1) and use the following formula to choose - * between percpu_counter and atomic_t. + * between the percpu counter and atomic_t. The batch has to be a power of 2. */ #define PERCPU_COUNTER_BATCH 32 - if (attr->max_entries / 2 > num_online_cpus() * PERCPU_COUNTER_BATCH) + if (attr->max_entries / 2 > nr_cpu_ids * PERCPU_COUNTER_BATCH) htab->use_percpu_counter = true; - if (htab->use_percpu_counter) { - err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL); - if (err) - goto free_map_locked; - } - if (prealloc) { err = prealloc_init(htab); if (err) @@ -789,8 +780,6 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr) free_prealloc: prealloc_destroy(htab); free_map_locked: - if (htab->use_percpu_counter) - percpu_counter_destroy(&htab->pcount); bpf_map_area_free(htab->buckets); bpf_mem_alloc_destroy(&htab->pcpu_ma); bpf_mem_alloc_destroy(&htab->ma); @@ -1162,30 +1151,33 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) static bool is_map_full(struct bpf_htab *htab) { - if (htab->use_percpu_counter) - return __percpu_counter_compare(&htab->pcount, htab->map.max_entries, - PERCPU_COUNTER_BATCH) >= 0; - return atomic_read(&htab->count) >= htab->map.max_entries; + s64 count = atomic_read(&htab->count); + + /* each cpu's elem_count is ahead of count by less than the batch */ + if (htab->use_percpu_counter && + count + nr_cpu_ids * PERCPU_COUNTER_BATCH >= htab->map.max_entries) + count = bpf_map_sum_elem_count(&htab->map); + return count >= htab->map.max_entries; } static void inc_elem_count(struct bpf_htab *htab) { - bpf_map_inc_elem_count(&htab->map); + s64 cnt = this_cpu_inc_return(*htab->map.elem_count); - if (htab->use_percpu_counter) - percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH); - else + if (!htab->use_percpu_counter) atomic_inc(&htab->count); + else if (!(cnt & (PERCPU_COUNTER_BATCH - 1))) + atomic_add(PERCPU_COUNTER_BATCH, &htab->count); } static void dec_elem_count(struct bpf_htab *htab) { - bpf_map_dec_elem_count(&htab->map); + s64 cnt = this_cpu_dec_return(*htab->map.elem_count); - if (htab->use_percpu_counter) - percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH); - else + if (!htab->use_percpu_counter) atomic_dec(&htab->count); + else if (!((cnt + 1) & (PERCPU_COUNTER_BATCH - 1))) + atomic_sub(PERCPU_COUNTER_BATCH, &htab->count); } static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) @@ -1866,8 +1858,6 @@ static void htab_map_free(struct bpf_map *map) bpf_map_area_free(htab->buckets); bpf_mem_alloc_destroy(&htab->pcpu_ma); bpf_mem_alloc_destroy(&htab->ma); - if (htab->use_percpu_counter) - percpu_counter_destroy(&htab->pcount); bpf_map_area_free(htab); } @@ -2577,9 +2567,7 @@ static u64 htab_map_mem_usage(const struct bpf_map *map) } else { #define LLIST_NODE_SZ sizeof(struct llist_node) - num_entries = htab->use_percpu_counter ? - percpu_counter_sum(&htab->pcount) : - atomic_read(&htab->count); + num_entries = bpf_map_sum_elem_count(map); usage += (htab->elem_size + LLIST_NODE_SZ) * num_entries; if (percpu) { usage += (LLIST_NODE_SZ + sizeof(void *)) * num_entries; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 58e357302850..6788489a2608 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1222,6 +1222,8 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) { struct bpf_hrtimer *t = container_of(hrtimer, struct bpf_hrtimer, timer); struct bpf_map *map = READ_ONCE(t->cb.map); + struct bpf_prog *prog = READ_ONCE(t->cb.prog); + struct bpf_timed_may_goto saved; void *value = t->cb.value; bpf_callback_t callback_fn; void *key; @@ -1229,7 +1231,7 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) BTF_TYPE_EMIT(struct bpf_timer); callback_fn = rcu_dereference_check(t->cb.callback_fn, rcu_read_lock_bh_held()); - if (!callback_fn || !map) + if (!callback_fn || !map || !prog) goto out; /* bpf_timer_cb() runs in hrtimer_run_softirq. It doesn't migrate and @@ -1242,8 +1244,10 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) key = map_key_from_value(map, value, &idx); + bpf_timed_may_goto_save(prog, &saved); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); /* The verifier checked that return value is zero. */ + bpf_timed_may_goto_restore(prog, &saved); this_cpu_write(hrtimer_running, NULL); out: @@ -1254,6 +1258,8 @@ static void bpf_wq_work(struct work_struct *work) { struct bpf_work *w = container_of(work, struct bpf_work, work); struct bpf_async_cb *cb = &w->cb; + struct bpf_timed_may_goto saved; + struct bpf_prog *prog; struct bpf_map *map; bpf_callback_t callback_fn; void *value = cb->value; @@ -1267,14 +1273,17 @@ static void bpf_wq_work(struct work_struct *work) map = READ_ONCE(cb->map); callback_fn = READ_ONCE(cb->callback_fn); - if (!callback_fn || !map) + prog = READ_ONCE(cb->prog); + if (!callback_fn || !map || !prog) goto out; key = map_key_from_value(map, value, &idx); migrate_disable(); + bpf_timed_may_goto_save(prog, &saved); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + bpf_timed_may_goto_restore(prog, &saved); migrate_enable(); out: @@ -3088,6 +3097,10 @@ __bpf_kfunc struct task_struct *bpf_task_from_vpid(s32 vpid) * bpf_dynptr_slice will not invalidate any ctx->data/data_end pointers in * the bpf program. * + * Please also note that when the buffer is on the stack, the verifier invalidates + * the returned pointer when the program stores to the buffer (helpers and kfuncs + * may fill it), or returns from the function that has it. + * * Return: NULL if the call failed (eg invalid dynptr), pointer to a read-only * data slice (can be either direct pointer to the data or a pointer to the user * provided buffer, with its contents containing the data, if unable to obtain @@ -3173,6 +3186,12 @@ __bpf_kfunc void *bpf_dynptr_slice(const struct bpf_dynptr *p, u64 offset, * Please note that, as in the example above, the user must check that the * returned pointer is not null before using it. * + * Please also note that when the buffer is on the stack, the verifier invalidates + * the returned pointer when the program stores to the buffer (helpers and kfuncs + * may fill it), or returns from the function that has it. Change the data through + * the returned pointer, and before the comparison with the buffer, as in the + * example above: after it the compiler may use one pointer in place of the other. + * * Please also note that in the case of skb and xdp dynptrs, bpf_dynptr_slice_rdwr * does not change the underlying packet data pointers, so a call to * bpf_dynptr_slice_rdwr will not invalidate any ctx->data/data_end pointers in @@ -3294,8 +3313,10 @@ __bpf_kfunc int bpf_dynptr_copy(const struct bpf_dynptr *dst_ptr, u64 dst_off, { const struct bpf_dynptr_kern *dst = (struct bpf_dynptr_kern *)dst_ptr; const struct bpf_dynptr_kern *src = (struct bpf_dynptr_kern *)src_ptr; + bool same = src->data == dst->data; void *src_slice, *dst_slice; char buf[256]; + bool rev; u64 off; src_slice = bpf_dynptr_slice(src_ptr, src_off, NULL, size); @@ -3306,25 +3327,31 @@ __bpf_kfunc int bpf_dynptr_copy(const struct bpf_dynptr *dst_ptr, u64 dst_off, return 0; } - if (src_slice) + /* The other range is copied piece by piece. A slice of the same packet + * may overlap with it, so it cannot be the buffer for that. + */ + if (src_slice && !same) return __bpf_dynptr_write(dst, dst_off, src_slice, size, 0); - if (dst_slice) + if (dst_slice && !same) return __bpf_dynptr_read(dst_slice, size, src, src_off, 0); if (bpf_dynptr_check_off_len(dst, dst_off, size) || bpf_dynptr_check_off_len(src, src_off, size)) return -E2BIG; + /* like memmove(): from the end if the destination is above the source */ + rev = same && dst->offset + dst_off > src->offset + src_off; off = 0; while (off < size) { u64 chunk_sz = min_t(u64, sizeof(buf), size - off); + u64 pos = rev ? size - off - chunk_sz : off; int err; - err = __bpf_dynptr_read(buf, chunk_sz, src, src_off + off, 0); + err = __bpf_dynptr_read(buf, chunk_sz, src, src_off + pos, 0); if (err) return err; - err = __bpf_dynptr_write(dst, dst_off + off, buf, chunk_sz, 0); + err = __bpf_dynptr_write(dst, dst_off + pos, buf, chunk_sz, 0); if (err) return err; @@ -4593,6 +4620,7 @@ static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx, static void bpf_task_work_callback(struct callback_head *cb) { struct bpf_task_work_ctx *ctx = container_of(cb, struct bpf_task_work_ctx, work); + struct bpf_timed_may_goto saved; enum bpf_task_work_state state; u32 idx; void *key; @@ -4614,8 +4642,10 @@ static void bpf_task_work_callback(struct callback_head *cb) key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx); migrate_disable(); + bpf_timed_may_goto_save(ctx->prog, &saved); ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key, (u64)(long)ctx->map_val, 0, 0); + bpf_timed_may_goto_restore(ctx->prog, &saved); migrate_enable(); bpf_task_work_ctx_reset(ctx); @@ -4855,6 +4885,7 @@ static void bpf_rcu_run_callback(struct rcu_head *rcu) struct bpf_rcu_head_kern *rh = container_of(rcu, struct bpf_rcu_head_kern, rcu); bpf_callback_t callback_fn = rh->callback_fn; struct bpf_prog *prog = rh->prog; + struct bpf_timed_may_goto saved; struct bpf_map *map = rh->map; void *value, *key; u32 idx; @@ -4866,7 +4897,9 @@ static void bpf_rcu_run_callback(struct rcu_head *rcu) smp_store_release(&rh->armed, 0); rcu_read_lock_dont_migrate(); + bpf_timed_may_goto_save(prog, &saved); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + bpf_timed_may_goto_restore(prog, &saved); rcu_read_unlock_migrate(); bpf_prog_put(prog); diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index e657e6d95408..75d9af4921de 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -976,6 +976,19 @@ static struct arg_track arg_join_imprecise(struct arg_track a, struct arg_track return (struct arg_track){ .mask = m, .frame = ARG_IMPRECISE }; } +/* + * 'a' or a value that is not a pointer to stack. A single fp-N gets a fake fp+0, + * to keep stack_use and prevent stack_def. fp+0 itself would stay a single offset. + */ +static struct arg_track arg_or_none(struct arg_track a) +{ + if (a.off_cnt != 1) + return a; + if (!a.off[0]) + return (struct arg_track){ .frame = a.frame }; + return arg_merge_offsets(a, arg_single(a.frame, 0)); +} + /* Join two arg_track values at merge points */ static struct arg_track __arg_track_join(struct arg_track a, struct arg_track b) { @@ -1000,20 +1013,10 @@ static struct arg_track __arg_track_join(struct arg_track a, struct arg_track b) */ if (a.frame == ARG_NONE && b.frame == ARG_NONE) return a; - if (a.frame >= 0 && b.frame == ARG_NONE) { - /* - * When joining single fp-N add fake fp+0 to - * keep stack_use and prevent stack_def - */ - if (a.off_cnt == 1) - return arg_merge_offsets(a, arg_single(a.frame, 0)); - return a; - } - if (b.frame >= 0 && a.frame == ARG_NONE) { - if (b.off_cnt == 1) - return arg_merge_offsets(b, arg_single(b.frame, 0)); - return b; - } + if (a.frame >= 0 && b.frame == ARG_NONE) + return arg_or_none(a); + if (b.frame >= 0 && a.frame == ARG_NONE) + return arg_or_none(b); return arg_join_imprecise(a, b); } @@ -1380,6 +1383,31 @@ static struct arg_track arg_track_deref(struct bpf_verifier_env *env, struct bpf return (struct arg_track){ .frame = ARG_NONE }; } +/* + * True if the call at @insn_idx can pass a pointer to the frame at @depth to a static subprog + * or to a callback. They use it as a pointer to stack, a global subprog gets plain memory. + * It errs on the safe side: it looks at all argument registers and stack args, also at those + * that this callee doesn't take. + */ +static bool subprog_gets_local_fp(struct bpf_verifier_env *env, struct bpf_insn *insn, int insn_idx, + struct arg_track *at, int depth) +{ + int r; + + if (bpf_pseudo_call(insn)) { + r = bpf_find_subprog(env, insn_idx + insn->imm + 1); + if (r >= 0 && bpf_subprog_is_global(env, r)) + return false; + } else if (!bpf_is_callx(insn) && !bpf_calls_callback(env, insn_idx)) { + return false; + } + /* R1-R5 and stack args */ + for (r = BPF_REG_1; r < MAX_AT_TRACK_REGS; r++) + if ((r <= BPF_REG_5 || r >= MAX_BPF_REG) && can_be_local_fp(depth, r, &at[r])) + return true; + return false; +} + /* * Pure dataflow transfer function for arg_track state. * Updates at_out[] based on how the instruction modifies registers. @@ -1443,21 +1471,22 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, *dst = none; } else if (class == BPF_JMP && code == BPF_CALL) { /* - * at_stack_out[slot] is not cleared by the helper and subprog calls. - * The fill_from_stack() may return the stale spill — which is an FP-derived arg_track - * (the value that was originally spilled there). The loaded register then carries - * a phantom FP-derived identity that doesn't correspond to what's actually in the slot. - * This phantom FP pointer propagates forward, and wherever it's subsequently used - * (as a helper argument, another store, etc.), it sets stack liveness bits. - * Those bits correspond to stack accesses that don't actually happen. - * So the effect is over-reporting stack liveness — marking slots as live that aren't - * actually accessed. The verifier preserves more state than necessary across calls, - * which is conservative. + * Helpers, kfuncs and global subprogs can scratch stack slots. The verifier reads + * a scalar from a scratched slot, but at_stack_out[slot] is not cleared, and + * fill_from_stack() returns the stale spill: an FP-derived arg_track for a scalar. + * A scalar can't be dereferenced, so the stale value can only set liveness bits + * for stack accesses that don't happen. That is conservative. * - * helpers can scratch stack slots, but they won't make a valid pointer out of it. - * subprogs are allowed to write into parent slots, but they cannot write - * _any_ FP-derived pointer into it (either their own or parent's FP). + * Static subprogs and callbacks can't store an FP-derived pointer into the frame of + * the caller, but they can store any other pointer there. The caller can fill it + * and store through it. With the stale spill that store would be a definite write + * to a stack slot, and it doesn't touch the stack. Callees are analyzed after the + * caller, and they reach the whole frame from one pointer to it. So after the call + * every slot holds what it held before or a value that is not FP-derived. */ + if (subprog_gets_local_fp(env, insn, insn_idx, at_out, depth)) + for (slot = 0; slot < nslots; slot++) + at_stack_out[slot] = __arg_track_join(at_stack_out[slot], none); for (r = BPF_REG_0; r <= BPF_REG_5; r++) at_out[r] = none; } else if (class == BPF_LDX) { diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index fa29d9b994e4..916408c63355 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -684,6 +684,8 @@ static void print_reg_state(struct bpf_verifier_env *env, if (base_type(t) == PTR_TO_MEM) { verbose_a("sz="); verbose_unum(env, reg->mem_size); + if (reg->buf.size) + verbose_a("buf=fp[%u]%d+%u", reg->buf.frameno, reg->buf.off, reg->buf.size); } if (t == CONST_PTR_TO_DYNPTR) verbose_a("type=%s", dynptr_type_str(reg->dynptr.type)); diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index 7b82fd79f9cc..06dee7e421ea 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -1831,7 +1831,11 @@ scev_reg_for_read(struct bpf_verifier_env *env, const struct bpf_func_state *st, return zero ? &env->scev->zero_reg : &env->scev->unknown_reg; } -/* Materialize a writable value in the current state, growing its stack if needed. */ +/* + * Materialize a writable value in the current state, growing its stack if needed. + * This does not go through check_stack_write(). The callers skip is_unbound_scev_reg() slots, + * thus the buffer of a live dynptr slice, see bpf_reg_state::buf, does not become a spill here. + */ static int scev_reg_for_write(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 r, struct bpf_reg_state **out) { @@ -2762,6 +2766,7 @@ static int union_any_reg(struct bpf_verifier_env *env, struct bpf_func_state *lo const struct bpf_reg_state *leaf_reg; struct scev *scev = env->scev; u32 l, r, leaf, order; + s64 smax = reg_smax(acc); s64 imm; int err; @@ -2785,10 +2790,13 @@ static int union_any_reg(struct bpf_verifier_env *env, struct bpf_func_state *lo verifier_bug(env, "scev ANY union has an unsupported leaf"); return -EFAULT; } + smax = max(smax, reg_smax(leaf_reg)); err = bpf_reg_union(env, acc, leaf_reg); if (err) return err; } + /* All leaves have the same range. It goes with the largest offset of all leaves. */ + bpf_trim_pkt_range(acc, smax); return 0; } @@ -2837,6 +2845,9 @@ int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state bpf_log(log, "\n"); } scratch_widened_reg_id(env, reg); + /* "at pkt_end" and "beyond pkt_end" are not valid in all iterations */ + if (type_is_pkt_pointer(reg->type)) + reg->range = max(reg->range, 0); err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, bounds.base, bounds.step); if (err) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index f7e8d766d128..a19126f5db02 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -522,12 +522,14 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map, u32 skip = flags & BPF_F_SKIP_FIELD_MASK; u32 max_depth; + max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); + /* trace_nr can be above max_depth and max_depth below skip: clamp first */ + trace_nr = min_t(u32, trace_nr, max_depth); if (trace_nr <= skip) /* skipping more than usable stack trace */ return -EFAULT; - max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); - stackid->nr = min_t(u32, trace_nr - skip, max_depth - skip); + stackid->nr = trace_nr - skip; stackid->len = stackid->nr * sizeof(u64); stackid->ips = trace->ip + skip; stackid->hash = jhash2((const u32 *)stackid->ips, stackid->len / sizeof(u32), 0); @@ -642,7 +644,8 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, /* couldn't fetch the stack trace */ return -EFAULT; - err = stackid_fastpath(&stackid, map, trace, trace->nr, flags); + /* the entry is released already, an NMI can change trace->nr */ + err = stackid_fastpath(&stackid, map, trace, READ_ONCE(trace->nr), flags); if (err != -ENOENT) return err; @@ -822,14 +825,20 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, crosstask, false, 0); } - if (unlikely(!trace) || trace->nr < skip) { - if (may_fault) - rcu_read_unlock(); - preempt_enable(); - goto err_fault; - } + if (unlikely(!trace)) + goto err_unlock; - trace_nr = callchain_store(trace, trace->nr, buf, elem_size, flags); + /* + * trace->nr can be above max_depth. The per-CPU entry is released + * already: a tracing program or an NMI which takes a callchain in the + * same context before the copy below reuses it. And the unwinders of + * PowerPC store context markers on top of max_depth. + */ + trace_nr = min_t(u32, READ_ONCE(trace->nr), max_depth); + if (trace_nr < skip) + goto err_unlock; + + trace_nr = callchain_store(trace, trace_nr, buf, elem_size, flags); /* trace should not be dereferenced after this point */ if (may_fault) @@ -838,6 +847,10 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, return callchain_finalize(buf, size, trace_nr, elem_size, flags, may_fault); +err_unlock: + if (may_fault) + rcu_read_unlock(); + preempt_enable(); err_fault: err = -EFAULT; clear: diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index a6ae86f973e3..50480c42bcad 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1116,6 +1116,20 @@ static int propagate_precision(struct bpf_verifier_env *env, bpf_bt_set_frame_slot(&env->bt, fr, i); first = false; } + for (i = 0; i < state->out_stack_arg_cnt; i++) { + state_reg = &state->stack_arg_regs[i]; + if (state_reg->type != SCALAR_VALUE || !state_reg->precise) + continue; + if (env->log.level & BPF_LOG_LEVEL2) { + if (first) + verbose(env, "frame %d: propagating r11%d", + fr, -(i + 1) * BPF_REG_SIZE); + else + verbose(env, ",r11%d", -(i + 1) * BPF_REG_SIZE); + } + bt_set_frame_stack_arg_slot(&env->bt, fr, i); + first = false; + } if (!first && (env->log.level & BPF_LOG_LEVEL2)) verbose(env, "\n"); } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 26a5ee0708a3..5e4700fda66b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1010,6 +1010,13 @@ static bool dynptr_write_changes_pkt_data(enum bpf_dynptr_type type) return type == BPF_DYNPTR_TYPE_SKB || type == BPF_DYNPTR_TYPE_SKB_META; } +/* The data of the other types is contiguous: bpf_dynptr_slice() does not use the buffer. */ +static bool dynptr_slice_may_be_buffer(enum bpf_dynptr_type type) +{ + return type != BPF_DYNPTR_TYPE_LOCAL && type != BPF_DYNPTR_TYPE_RINGBUF && + type != BPF_DYNPTR_TYPE_SKB_META; +} + static bool is_dynptr_type_expected(struct bpf_verifier_env *env, struct bpf_reg_state *reg, enum bpf_arg_type arg_type) { @@ -4558,6 +4565,31 @@ static int check_stack_read(struct bpf_verifier_env *env, return err; } +/* + * An access through a slice that may be the buffer on the stack, see bpf_reg_state::buf, is + * not seen as an access to the stack. That's fine as long as nothing is tracked in the buffer. + * The output of helpers and kfuncs is not tracked. Here an insn stores to [min_off, max_off) + * of frame @frameno, a dynptr, an iterator or irq flags are made there, or the frame is gone. + */ +static void invalidate_stack_slices(struct bpf_verifier_env *env, u32 frameno, + int min_off, int max_off) +{ + struct bpf_func_state *state; + struct bpf_reg_state *reg; + + if (!env->seen_stack_slice) + return; + + bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ + if (base_type(reg->type) != PTR_TO_MEM || !reg->buf.size || + reg->buf.frameno != frameno || reg->buf.off >= max_off || + reg->buf.off + (int)reg->buf.size <= min_off) + continue; + bpf_diag_record_scrub(env, reg, BPF_DIAG_MOD_SLICE_BUF); + mark_reg_invalid(env, reg); + })); +} + /* check_stack_write dispatches to check_stack_write_fixed_off or * check_stack_write_var_off. * @@ -4572,6 +4604,8 @@ static int check_stack_write(struct bpf_verifier_env *env, int value_regno, int insn_idx) { struct bpf_func_state *state = bpf_func(env, reg); + int min_off = reg_smin(reg) + off; + int max_off = reg_smax(reg) + off + size; int err; if (tnum_is_const(reg->var_off)) { @@ -4586,6 +4620,8 @@ static int check_stack_write(struct bpf_verifier_env *env, reg, off, size, value_regno, insn_idx); } + /* After the write is modelled: the insn may store a slice to its own buffer. */ + invalidate_stack_slices(env, state->frameno, min_off, max_off); return err; } @@ -10466,15 +10502,18 @@ static int mark_raw_stack(struct bpf_verifier_env *env, struct bpf_call_arg_meta */ for (slot = 0; slot < MAX_BPF_FUNC_ARGS; slot++) { struct bpf_reg_state *reg; - argno_t argno = argno_from_arg(slot + 1); if (!meta->arg_raw_mem.size[slot]) continue; reg = get_func_arg_reg(caller, cur_regs(env), slot); + /* + * check_stack_range_initialized() checked the range, its offset is a constant. + * Not check_stack_write(): unknown bytes in its buffer leave a dynptr slice valid. + */ for (i = 0; i < meta->arg_raw_mem.size[slot]; i++) { - err = check_mem_access(env, insn_idx, reg, argno, i, BPF_B, - BPF_WRITE, -1, false, false); + err = check_stack_write_fixed_off(env, bpf_func(env, reg), + reg->var_off.value + i, 1, -1, insn_idx); if (err) return err; } @@ -11469,6 +11508,12 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins if (err) return err; + if (!bpf_pseudo_kfunc_call(insn) && + (insn->imm == BPF_FUNC_loop || insn->imm == BPF_FUNC_for_each_map_elem)) + env->subprog_info[subprog].is_loop_cb = true; + else + env->subprog_info[subprog].is_called = true; + /* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ @@ -11505,6 +11550,7 @@ static int check_static_func_call(struct bpf_verifier_env *env, int subprog, int err; caller = state->frame[state->curframe]; + env->subprog_info[subprog].is_called = true; /* * Track caller's total stack arg count (incoming + max outgoing). @@ -12176,6 +12222,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) callee->allocated_stack / BPF_REG_SIZE - 1); if (err) return err; + invalidate_stack_slices(env, callee->frameno, INT_MIN, 0); /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, * there function call logic would reschedule callback visit. If iteration @@ -13574,7 +13621,12 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env, if (base_type(*arg_type) != ARG_PTR_TO_BTF_ID) return 0; + /* + * A kfunc that releases an object, or that takes a reference to one, + * needs the object itself: memory of the program cannot stand in for it. + */ if (!is_kfunc(meta) || arg_type_is_release(*arg_type) || + is_kfunc_acquire(meta) || base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)]) return 0; @@ -13585,7 +13637,15 @@ static int resolve_func_arg_type(struct bpf_verifier_env *env, ref_t = btf_type_skip_modifiers(btf, ref_id, &ref_id); ref_tname = btf_name_by_offset(btf, ref_t->name_off); - if (!btf_type_is_scalar_struct(env, btf, ref_t)) + /* + * A kfunc that takes a projection type (ie. xdp_md) casts it to the + * underlying type. So memory of the program cannot stand in for it, + * even if the projection type holds only scalars. A struct without + * members (ie. iou_ctx) is a handle for an object of another type. + */ + if (btf_is_projection_type(ref_tname) || + !btf_type_is_scalar_struct(env, btf, ref_t) || + !btf_type_vlen(ref_t)) return 0; resolve_ret = btf_resolve_size(btf, ref_t, arg_size); @@ -14340,9 +14400,13 @@ static int process_arg_ptr_to_btf_id(struct bpf_verifier_env *env, struct bpf_re /* If kfunc is accepting a projection type (ie. __sk_buff), it cannot * actually use it -- it must cast to the underlying type. So we allow - * caller to pass in the underlying type. + * caller to pass in the underlying type. Only kernel BTF has that type: + * a type of the same name in the BTF of the program is program memory. + * And only a pointer to the start of the object points to that type. */ - taking_projection = is_kfunc(meta) && btf_is_projection_of(arg_tname, reg_tname); + taking_projection = is_kfunc(meta) && btf_is_kernel(reg_btf) && + !reg->var_off.value && + btf_is_projection_of(arg_tname, reg_tname); if (!taking_projection && !struct_same) { verbose(env, "%s %s expected pointer to %s %s but %s has a pointer to %s %s\n", meta->func_name, reg_arg_name(env, argno), @@ -15394,6 +15458,9 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg return -EFAULT; } regs[BPF_REG_0].parent_id = meta->dynptr.id; + regs[BPF_REG_0].buf = meta->slice_buf; + if (meta->slice_buf.size) + env->seen_stack_slice = true; } else { return 0; } @@ -15651,6 +15718,22 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, ref_convert_owning_non_owning(env, id); } + if ((is_kfunc_call(&meta, special_kfunc_list[KF_bpf_dynptr_slice]) || + is_kfunc_call(&meta, special_kfunc_list[KF_bpf_dynptr_slice_rdwr])) && + dynptr_slice_may_be_buffer(meta.dynptr.type)) { + struct bpf_reg_state *buf = ®s[BPF_REG_3]; + + if (buf->type == PTR_TO_STACK) { + meta.slice_buf.frameno = buf->frameno; + meta.slice_buf.off = reg_smin(buf); + meta.slice_buf.size = reg_smax(buf) - reg_smin(buf) + + meta.arg_constant.value; + } else if (base_type(buf->type) == PTR_TO_MEM) { + /* a slice is the buffer of the next one */ + meta.slice_buf = buf->buf; + } + } + if (meta.func_id == special_kfunc_list[KF_bpf_throw]) { if (!bpf_jit_supports_exceptions()) { verbose(env, "JIT does not support calling kfunc %s#%d\n", @@ -17736,6 +17819,18 @@ bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, } } +/* + * 'range - smax' bytes from a packet pointer are in the packet. reg_bounds_sync() can make smax + * smaller than @smax: with a step of 8, or with 3 zero bits in var_off, 58 becomes 56. Make the + * range smaller by the same number, so that the number of bytes does not grow. umax is not + * sufficient: it is U64_MAX when the offset can be negative. + */ +void bpf_trim_pkt_range(struct bpf_reg_state *reg, s64 smax) +{ + if (type_is_pkt_pointer(reg->type) && reg->range > 0) + reg->range = max_t(s64, reg->range - (smax - reg_smax(reg)), 0); +} + int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, struct cnum64 r64, struct cnum32 r32, u16 base, u16 step) { @@ -17745,6 +17840,7 @@ int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, reg->base = base; reg->var_off = tnum_unknown; reg_bounds_sync(reg); /* this should infer the tnum alignment */ + bpf_trim_pkt_range(reg, cnum64_smax(r64)); return reg_bounds_sanity_check(env, reg, "bpf_set_reg_range"); } @@ -22497,6 +22593,37 @@ static bool func_has_rcu_read_lock(const struct bpf_prog *prog, int subprog) !sub_aux->called[BPF_FUNC_CTX_NO_RCU_LOCK]; } +/* + * The callers of a global function are checked against the arguments that its + * BTF describes, and an extension is verified with those of its own BTF. It + * must not expect more than the callers of the function provide. + * + * While an extension is loaded, bpf_check_attach_target() runs before its + * arguments are known. check_attach_btf_id() does the check when they are. + */ +static int check_ext_args(struct bpf_verifier_log *log, const struct bpf_prog *prog, + const struct bpf_prog *tgt_prog, int subprog) +{ + const struct bpf_subprog_arg_info *tgt = tgt_prog->aux->func_info_aux[subprog].args; + const struct bpf_subprog_arg_info *ext = prog->aux->func_info_aux[0].args; + int i; + + for (i = 0; i < MAX_BPF_FUNC_REG_ARGS && ext[i].arg_type != ARG_UNUSED; i++) { + /* An extension may accept NULL where the callers never pass it. */ + if ((ext[i].arg_type == tgt[i].arg_type || + ext[i].arg_type == (tgt[i].arg_type | PTR_MAYBE_NULL)) && + (base_type(ext[i].arg_type) == ARG_PTR_TO_MEM ? + ext[i].mem_size <= tgt[i].mem_size : ext[i].btf_id == tgt[i].btf_id)) + continue; + + bpf_log(log, "Extension program's argument in R%d is incompatible with original\n", + i + 1); + return -EINVAL; + } + + return 0; +} + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -22784,7 +22911,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, return -EINVAL; } if (prog_extension && - btf_check_type_match(log, prog, btf, t)) + (btf_check_type_match(log, prog, btf, t) || + check_ext_args(log, prog, tgt_prog, subprog))) return -EINVAL; t = btf_attach_func_proto(log, btf, btf_id); if (!t || !btf_type_is_func_proto(t)) @@ -23004,6 +23132,15 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) */ env->ops = bpf_verifier_ops[tgt_prog->type]; prog->expected_attach_type = tgt_prog->expected_attach_type; + + /* only now it is known what the arguments of prog stand for */ + ret = btf_prepare_func_args(env, 0); + if (ret) + return ret; + ret = check_ext_args(&env->log, prog, tgt_prog, + find_tgt_subprog(tgt_prog, btf_id)); + if (ret) + return ret; } /* store info about the attachment target that will be used later */ diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 3043a7393c29..d6c9dc62182e 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -1210,7 +1210,8 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, skb->ip_summed = CHECKSUM_COMPLETE; } - if (prog->type == BPF_PROG_TYPE_LWT_XMIT) { + if (prog->type == BPF_PROG_TYPE_LWT_IN || + prog->type == BPF_PROG_TYPE_LWT_XMIT) { if (!ipv6_mod_enabled()) { pr_warn_once("Please test this program with IPv6 enabled kernel\n"); ret = -EOPNOTSUPP; diff --git a/net/core/filter.c b/net/core/filter.c index 09c60a58c82a..9245009d9f1e 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -2290,9 +2290,13 @@ static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb, static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev, struct bpf_nh_params *nh) { - const struct ipv6hdr *ip6h = ipv6_hdr(skb); struct net *net = dev_net(dev); int err, ret = NET_XMIT_DROP; + const struct ipv6hdr *ip6h; + + if (unlikely(!pskb_may_pull(skb, sizeof(*ip6h)))) + goto out_drop; + ip6h = ipv6_hdr(skb); if (!nh) { struct dst_entry *dst; @@ -2399,9 +2403,13 @@ static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb, static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev, struct bpf_nh_params *nh) { - const struct iphdr *ip4h = ip_hdr(skb); struct net *net = dev_net(dev); int err, ret = NET_XMIT_DROP; + const struct iphdr *ip4h; + + if (unlikely(!pskb_may_pull(skb, sizeof(*ip4h)))) + goto out_drop; + ip4h = ip_hdr(skb); if (!nh) { struct flowi4 fl4 = { @@ -2451,15 +2459,14 @@ static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev, static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev, struct bpf_nh_params *nh) { - struct ethhdr *ethh = eth_hdr(skb); - - if (unlikely(skb->mac_header >= skb->network_header)) + if (unlikely(skb_network_offset(skb) < ETH_HLEN || + !pskb_may_pull(skb, ETH_HLEN))) goto out; bpf_push_mac_rcsum(skb); - if (is_multicast_ether_addr(ethh->h_dest)) + if (is_multicast_ether_addr(eth_hdr(skb)->h_dest)) goto out; - skb_pull(skb, sizeof(*ethh)); + skb_pull(skb, ETH_HLEN); skb_unset_mac_header(skb); skb_reset_network_header(skb); @@ -3362,6 +3369,10 @@ static const struct bpf_func_proto bpf_skb_vlan_pop_proto = { static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len) { + /* The bytes in front of @off are moved with skb->data. */ + if (unlikely(off > skb->len || !pskb_may_pull(skb, off))) + return -ENOMEM; + /* Caller already did skb_cow() with meta_len+len as headroom, * so no need to do it here. */ @@ -3382,9 +3393,10 @@ static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len) void *old_data; /* skb_ensure_writable() is not needed here, as we're - * already working on an uncloned skb. + * already working on an uncloned skb. The first test + * keeps off + len from wrapping. */ - if (unlikely(!pskb_may_pull(skb, off + len))) + if (unlikely(off > skb->len || !pskb_may_pull(skb, off + len))) return -ENOMEM; old_data = skb->data; @@ -3395,9 +3407,25 @@ static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len) return 0; } +/* + * bpf_skb_net_hdr_push() inserts bytes at @off, bpf_skb_net_hdr_pop() removes + * bytes at @off. Both move the bytes in front of @off together with skb->data, + * and leave the rest of the packet where it is. Return true if the transport + * header is in the part that moves. A transport header that is equal to the + * network header stays equal to it. + */ +static bool bpf_skb_trans_hdr_moves(const struct sk_buff *skb, u32 off) +{ + if (skb->transport_header == skb->network_header) + return true; + + return skb_transport_header_was_set(skb) && + (u32)skb_transport_offset(skb) < off; +} + static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len) { - bool trans_same = skb->transport_header == skb->network_header; + bool trans_moves = bpf_skb_trans_hdr_moves(skb, off); int ret; /* There's no need for __skb_push()/__skb_pull() pair to @@ -3408,8 +3436,8 @@ static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len) if (likely(!ret)) { skb->mac_header -= len; skb->network_header -= len; - if (trans_same) - skb->transport_header = skb->network_header; + if (trans_moves) + skb->transport_header -= len; } return ret; @@ -3417,7 +3445,7 @@ static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len) static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len) { - bool trans_same = skb->transport_header == skb->network_header; + bool trans_moves = bpf_skb_trans_hdr_moves(skb, off); int ret; /* Same here, __skb_push()/__skb_pull() pair not needed. */ @@ -3425,8 +3453,17 @@ static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len) if (likely(!ret)) { skb->mac_header += len; skb->network_header += len; - if (trans_same) - skb->transport_header = skb->network_header; + if (trans_moves) { + skb->transport_header += len; + } else if (skb_transport_header_was_set(skb) && + skb_transport_offset(skb) < (int)off) { + /* It was in the removed bytes or before skb->data. */ + skb_set_transport_header(skb, off); + } + /* Do not leave the inner transport header in front of @off. */ + if (skb->encapsulation && + skb_inner_transport_offset(skb) < (int)off) + skb_set_inner_transport_header(skb, off); } return ret; @@ -3737,6 +3774,25 @@ static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff, return 0; } +/* + * Code that went by the inner transport header while skb->encapsulation was set + * goes by the transport header from now on. So take the inner one over, unless + * it is the smaller one, as it is if nobody has set it. Do not leave the result + * in front of @off, where the bytes were removed. + */ +static void bpf_skb_clear_encap(struct sk_buff *skb, u32 off) +{ + int thoff; + + if (!skb->encapsulation) + return; + + thoff = max(skb->inner_transport_header, skb->transport_header) - + skb_headroom(skb); + skb_set_transport_header(skb, max_t(int, thoff, off)); + skb->encapsulation = 0; +} + static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, u64 flags) { @@ -3755,6 +3811,14 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, return -ENOTSUPP; } + /* + * The MSS is raised by len_diff. It has 16 bits, and GSO_BY_FRAGS is + * not a size. + */ + if (skb_is_gso(skb) && !(flags & BPF_F_ADJ_ROOM_FIXED_GSO) && + skb_shinfo(skb)->gso_size + len_diff >= GSO_BY_FRAGS) + return -ENOTSUPP; + ret = skb_unclone(skb, GFP_ATOMIC); if (unlikely(ret < 0)) return ret; @@ -3810,8 +3874,7 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6 | SKB_GSO_ESP))) - if (skb->encapsulation) - skb->encapsulation = 0; + bpf_skb_clear_encap(skb, off); } /* Header must be checked, and gso_segs recomputed. */ @@ -3819,10 +3882,9 @@ static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff, shinfo->gso_segs = 0; } else { /* For non-GSO packets, clear encapsulation if decap flags are set */ - if ((flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | - BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) && - skb->encapsulation) - skb->encapsulation = 0; + if (flags & (BPF_F_ADJ_ROOM_DECAP_L4_MASK | + BPF_F_ADJ_ROOM_DECAP_IPXIP_MASK)) + bpf_skb_clear_encap(skb, off); } return 0; @@ -6909,14 +6971,22 @@ static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len int err; struct ipv6_sr_hdr *srh = (struct ipv6_sr_hdr *)hdr; + /* With ipv6.disable=1 the netns has no seg6 data and no IPv6 tables. */ + if (!ipv6_mod_enabled()) + return -EAFNOSUPPORT; + if (!seg6_validate_srh(srh, len, false)) return -EINVAL; + /* + * The encap mode names IPv6 as the inner protocol, and neither mode + * updates skb->protocol. + */ + if (skb->protocol != htons(ETH_P_IPV6)) + return -EBADMSG; + switch (type) { case BPF_LWT_ENCAP_SEG6_INLINE: - if (skb->protocol != htons(ETH_P_IPV6)) - return -EBADMSG; - err = seg6_do_srh_inline(skb, srh); break; case BPF_LWT_ENCAP_SEG6: @@ -7005,12 +7075,13 @@ BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset, this_cpu_ptr(&seg6_bpf_srh_states); struct ipv6_sr_hdr *srh = srh_state->srh; void *srh_tlvs, *srh_end, *ptr; - int srhoff = 0; + int srhoff; lockdep_assert_held(&srh_state->bh_lock); if (srh == NULL) return -EINVAL; + srhoff = (unsigned char *)srh - skb->data; srh_tlvs = (void *)((char *)srh + ((srh->first_segment + 1) << 4)); srh_end = (void *)((char *)srh + sizeof(*srh) + srh_state->hdrlen); @@ -7023,8 +7094,7 @@ BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset, if (unlikely(bpf_try_make_writable(skb, offset + len))) return -EFAULT; - if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0) - return -EINVAL; + /* The head may have moved. */ srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff); memcpy(skb->data + offset, from, len); @@ -7140,16 +7210,18 @@ BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset, { struct seg6_bpf_srh_state *srh_state = this_cpu_ptr(&seg6_bpf_srh_states); + bool mac_set = skb_mac_header_was_set(skb); struct ipv6_sr_hdr *srh = srh_state->srh; void *srh_end, *srh_tlvs, *ptr; struct ipv6hdr *hdr; - int srhoff = 0; + int srhoff; int ret; lockdep_assert_held(&srh_state->bh_lock); if (unlikely(srh == NULL)) return -EINVAL; + srhoff = (unsigned char *)srh - skb->data; srh_tlvs = (void *)((unsigned char *)srh + sizeof(*srh) + ((srh->first_segment + 1) << 4)); srh_end = (void *)((unsigned char *)srh + sizeof(*srh) + @@ -7160,26 +7232,46 @@ BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset, return -EFAULT; if (unlikely(len < 0 && (void *)((char *)ptr - len) > srh_end)) return -EFAULT; + /* The length field of the SRH has 8 bits, in units of 8 bytes. */ + if (unlikely(len > (U8_MAX << 3) - srh_state->hdrlen)) + return -EFAULT; + /* + * skb->data is the network header here. The MAC header is in front of + * it, so bpf_skb_net_hdr_push() and bpf_skb_net_hdr_pop() do not move + * it with the other headers. + */ if (len > 0) { - ret = skb_cow_head(skb, len); + ret = skb_cow_head(skb, len + skb->mac_len); if (unlikely(ret < 0)) return ret; + /* The push can write over the MAC header, so move it first. */ + if (mac_set) + memmove(skb->data - len - skb->mac_len, + skb_mac_header(skb), skb->mac_len); ret = bpf_skb_net_hdr_push(skb, offset, len); } else { ret = bpf_skb_net_hdr_pop(skb, offset, -1 * len); + /* The pop has added -len to skb->mac_header. */ + if (likely(!ret) && mac_set) + memmove(skb->data - skb->mac_len, + skb_mac_header(skb) + len, skb->mac_len); } bpf_compute_data_pointers(skb); if (unlikely(ret < 0)) return ret; + if (mac_set) + skb_set_mac_header(skb, -skb->mac_len); + else + skb_unset_mac_header(skb); + hdr = (struct ipv6hdr *)skb->data; hdr->payload_len = htons(skb->len - sizeof(struct ipv6hdr)); - if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0) - return -EINVAL; + /* The bytes in front of offset have moved together with skb->data. */ srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff); srh_state->hdrlen += len; srh_state->valid = false; diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index da49364ec63d..72c86ff2a847 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -85,6 +85,32 @@ static int run_lwt_bpf(struct sk_buff *skb, struct bpf_lwt_prog *lwt, return ret; } +/* + * skb->cb still describes the IP header that was the first one before the + * prog ran. The option offsets in IPCB(skb)->opt are wrong for a pushed + * header, and a header of the other family needs the other layout. iif + * stays: behind a VRF skb->skb_iif is the L3 master and not the port. + * bpf_lwt_push_encap() cannot do this, because bpf_prog_run_save_cb() can + * restore skb->cb after the prog. + */ +static void bpf_lwt_reset_cb(struct sk_buff *skb, __be16 proto) +{ + int iif; + + if (skb->protocol == proto) { + if (proto == htons(ETH_P_IP)) + memset(&IPCB(skb)->opt, 0, sizeof(IPCB(skb)->opt)); + } else if (skb->protocol == htons(ETH_P_IP)) { + iif = IP6CB(skb)->iif; + memset(IPCB(skb), 0, sizeof(*IPCB(skb))); + IPCB(skb)->iif = iif; + } else { + iif = IPCB(skb)->iif; + memset(IP6CB(skb), 0, sizeof(*IP6CB(skb))); + IP6CB(skb)->iif = iif; + } +} + static int bpf_lwt_input_reroute(struct sk_buff *skb) { enum skb_drop_reason reason; @@ -102,7 +128,7 @@ static int bpf_lwt_input_reroute(struct sk_buff *skb) dev_put(dev); } else if (skb->protocol == htons(ETH_P_IPV6)) { skb_dst_drop(skb); - if (IS_ENABLED(CONFIG_IPV6)) { + if (ipv6_mod_enabled()) { ip6_route_input(skb); err = skb_dst(skb)->error; } else { @@ -124,6 +150,8 @@ static int bpf_lwt_input_reroute(struct sk_buff *skb) static int bpf_input(struct sk_buff *skb) { struct dst_entry *dst = skb_dst(skb); + __be16 proto = skb->protocol; + unsigned int len = skb->len; struct bpf_lwt *bpf; int ret; @@ -132,8 +160,21 @@ static int bpf_input(struct sk_buff *skb) ret = run_lwt_bpf(skb, &bpf->in, dst, NO_REDIRECT); if (ret < 0) return ret; + /* An lwt_in prog can change the length only with a push. */ + if (skb->len != len) + bpf_lwt_reset_cb(skb, proto); if (ret == BPF_LWT_REROUTE) return bpf_lwt_input_reroute(skb); + /* + * orig_input() is for the old protocol only, see bpf_xmit(). + * It also needs a dst of the old family. BPF_LWT_ENCAP_SEG6 + * puts an IPv6 dst into the skb. + */ + if (skb->protocol != proto || + skb_dst(skb)->ops->family != dst->ops->family) { + kfree_skb(skb); + return -EINVAL; + } } if (unlikely(!dst->lwtstate->orig_input)) { @@ -304,13 +345,17 @@ static int bpf_xmit(struct sk_buff *skb) * made before LWT xmit. */ ret = xmit_check_headroom(skb, hroom); - if (unlikely(ret)) + if (unlikely(ret)) { + kfree_skb(skb); return ret; + } return LWTUNNEL_XMIT_CONTINUE; case BPF_REDIRECT: return LWTUNNEL_XMIT_DONE; case BPF_LWT_REROUTE: + /* An lwt_xmit prog can also write to the options. */ + bpf_lwt_reset_cb(skb, proto); return bpf_lwt_xmit_reroute(skb); default: return ret; @@ -649,6 +694,14 @@ int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress) if (ingress) skb_postpush_rcsum(skb, iph, len); skb_reset_network_header(skb); + /* + * The new header goes over the MAC header. On ingress move it first. + * On egress the packet gets a new one, so forget the old one. + */ + if (ingress) + skb_mac_header_rebuild(skb); + else + skb_unset_mac_header(skb); if (is_udp_tunnel) { size_t iph_sz = ipv4 ? iph->ihl * 4 : sizeof(struct ipv6hdr); diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 057e2f3c0265..efe49551caba 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -152,11 +152,11 @@ static void sock_map_del_link(struct sock *sk, struct bpf_map *map = link->map; struct sk_psock_progs *progs = sock_map_progs(map); - if (psock->saved_data_ready && progs->stream_parser) + if (progs->stream_parser) strp_stop = true; - if (psock->saved_data_ready && progs->stream_verdict) + if (progs->stream_verdict) verdict_stop = true; - if (psock->saved_data_ready && progs->skb_verdict) + if (progs->skb_verdict) verdict_stop = true; list_del(&link->list); sk_psock_free_link(link); @@ -164,8 +164,11 @@ static void sock_map_del_link(struct sock *sk, } } spin_unlock_bh(&psock->link_lock); - if (strp_stop || verdict_stop) { - write_lock_bh(&sk->sk_callback_lock); + if (!strp_stop && !verdict_stop) + return; + + write_lock_bh(&sk->sk_callback_lock); + if (psock->saved_data_ready) { if (strp_stop) sk_psock_stop_strp(sk, psock); if (verdict_stop) @@ -173,8 +176,8 @@ static void sock_map_del_link(struct sock *sk, if (psock->psock_update_sk_prot) psock->psock_update_sk_prot(sk, psock, false); - write_unlock_bh(&sk->sk_callback_lock); } + write_unlock_bh(&sk->sk_callback_lock); } static void sock_map_unref(struct sock *sk, void *link_raw) @@ -202,7 +205,7 @@ static struct sk_psock *sock_map_psock_get_checked(struct sock *sk) rcu_read_lock(); psock = sk_psock(sk); if (psock) { - if (sk->sk_prot->close != sock_map_close) { + if (READ_ONCE(sk->sk_prot)->close != sock_map_close) { psock = ERR_PTR(-EBUSY); goto out; } @@ -284,6 +287,7 @@ static int sock_map_link(struct bpf_map *map, struct sock *sk) } } + write_lock_bh(&sk->sk_callback_lock); if (msg_parser) psock_set_prog(&psock->progs.msg_parser, msg_parser); if (stream_parser) @@ -298,11 +302,11 @@ static int sock_map_link(struct bpf_map *map, struct sock *sk) */ ret = sock_map_init_proto(sk, psock); if (ret < 0) { + write_unlock_bh(&sk->sk_callback_lock); sk_psock_put(sk, psock); goto out; } - write_lock_bh(&sk->sk_callback_lock); if (stream_parser && stream_verdict && !psock->saved_data_ready) { if (sk_is_tcp(sk)) ret = sk_psock_init_strp(sk, psock); @@ -536,7 +540,7 @@ static bool sock_map_redirect_allowed(const struct sock *sk) static bool sock_map_sk_is_suitable(const struct sock *sk) { - return !!sk->sk_prot->psock_update_sk_prot; + return !!READ_ONCE(sk->sk_prot)->psock_update_sk_prot; } static bool sock_map_sk_state_allowed(const struct sock *sk) diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index d2ff6f5d6d5b..95c31803e722 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -2222,6 +2222,7 @@ struct btf_pipe { static int btf_rewrite_str(struct btf_pipe *p, __u32 *str_off) { + struct btf *base; long mapped_off; int off, err; @@ -2234,6 +2235,19 @@ static int btf_rewrite_str(struct btf_pipe *p, __u32 *str_off) return 0; } + /* + * btf__add_str() first looks the string up in the base BTFs of dst, + * which makes each of them modifiable and frees its raw data. If src + * shares a base with dst and the string lives in that base, the + * pointer taken below would be left dangling. So make the bases + * modifiable before the pointer is taken. + */ + for (base = p->dst->base_btf; base; base = base->base_btf) { + err = btf_ensure_modifiable(base); + if (err) + return err; + } + off = btf__add_str(p->dst, btf__str_by_offset(p->src, *str_off)); if (off < 0) return off; diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c index ea1fac95b461..017add229bef 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c @@ -11,6 +11,7 @@ #define TEST_BITMAP_BITS (2 * BITS_PER_LONG_LONG) #define TEST_BITMAP_SYNC_SPINS BPF_MAX_LOOPS #define TEST_BITMAP_ITERS 10 * 1000 * 1000 +#define TEST_BITMAP_NSEC (50 * 1000 * 1000) static struct arena_bitmap __arena *bitmap; static volatile u64 started; @@ -129,6 +130,7 @@ static int bitmap_test_common_single(int parity) static int bitmap_test_common(int parity) { + u64 start; int ret; u32 i; @@ -138,6 +140,7 @@ static int bitmap_test_common(int parity) if (ret) return ret; + start = bpf_ktime_get_ns(); for (i = zero; i < TEST_BITMAP_ITERS && can_loop; i++) { ret = bitmap_test_common_single(parity); if (ret) @@ -147,6 +150,13 @@ static int bitmap_test_common(int parity) break; bitmap_test_reset_single(parity); + + /* + * Stop before the time slice of may_goto is used up. After that the + * bmp_*() functions can't retry a cmpxchg that fails. + */ + if (bpf_ktime_get_ns() - start > TEST_BITMAP_NSEC) + break; } return 0; diff --git a/tools/testing/selftests/bpf/prog_tests/arena_mmap.c b/tools/testing/selftests/bpf/prog_tests/arena_mmap.c new file mode 100644 index 000000000000..f55d950720f1 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/arena_mmap.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#define ARENA_PAGES 8 + +/* err == 0: the range ends above TASK_SIZE. EINVAL or ENOMEM, this depends on the arch */ +static void test_refused(__u64 map_extra, __u64 addr, int pages, int flags, int err) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts, + .map_flags = BPF_F_MMAPABLE, + .map_extra = map_extra); + void *area; + int fd; + + fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, ARENA_PAGES, &opts); + if (fd == -EOPNOTSUPP) { + test__skip(); + return; + } + /* No task can map this range, so the kernel can also refuse the map. */ + if (fd < 0 && map_extra) + return; + if (!ASSERT_GE(fd, 0, "bpf_map_create")) + return; + + area = mmap((void *)(unsigned long)addr, pages * getpagesize(), + PROT_READ | PROT_WRITE, MAP_SHARED | flags, fd, 0); + if (!ASSERT_EQ(area, MAP_FAILED, "mmap")) + munmap(area, pages * getpagesize()); + else if (err || errno != ENOMEM) + ASSERT_EQ(errno, err ?: EINVAL, "errno"); + close(fd); +} + +void serial_test_arena_mmap(void) +{ + __u64 page = getpagesize(), last = -(ARENA_PAGES * page); + + if (test__start_subtest("long")) + test_refused(0, 0, ARENA_PAGES + 1, 0, E2BIG); + if (test__start_subtest("address_0")) + test_refused(0, 0, ARENA_PAGES, MAP_FIXED, EINVAL); + if (test__start_subtest("crosses_4g")) + test_refused(0, (1ULL << 32) - page, ARENA_PAGES, MAP_FIXED, EINVAL); + if (test__start_subtest("crosses_4g_noreplace")) + test_refused(0, (1ULL << 32) - page, ARENA_PAGES, MAP_FIXED_NOREPLACE, EINVAL); + if (test__start_subtest("wraps")) + test_refused(0, -page, ARENA_PAGES, MAP_FIXED, 0); + if (test__start_subtest("map_extra_last_pages")) + test_refused(last, last, ARENA_PAGES, MAP_FIXED, 0); +} diff --git a/tools/testing/selftests/bpf/prog_tests/dynptr.c b/tools/testing/selftests/bpf/prog_tests/dynptr.c index 4396560365e8..19f38215d909 100644 --- a/tools/testing/selftests/bpf/prog_tests/dynptr.c +++ b/tools/testing/selftests/bpf/prog_tests/dynptr.c @@ -22,6 +22,9 @@ static struct { {"test_dynptr_data", SETUP_SYSCALL_SLEEP}, {"test_dynptr_copy", SETUP_SYSCALL_SLEEP}, {"test_dynptr_copy_xdp", SETUP_XDP_PROG}, + {"test_dynptr_copy_xdp_overlap", SETUP_XDP_PROG}, + {"test_dynptr_copy_xdp_overlap_src_linear", SETUP_XDP_PROG}, + {"test_dynptr_copy_xdp_overlap_dst_frag", SETUP_XDP_PROG}, {"test_dynptr_memset_zero", SETUP_SYSCALL_SLEEP}, {"test_dynptr_memset_notzero", SETUP_SYSCALL_SLEEP}, {"test_dynptr_memset_zero_offset", SETUP_SYSCALL_SLEEP}, @@ -34,6 +37,7 @@ static struct { {"test_skb_readonly", SETUP_SKB_PROG}, {"test_dynptr_skb_data", SETUP_SKB_PROG}, {"test_dynptr_skb_slice_non_linear", SETUP_SKB_PROG_NONLINEAR}, + {"test_dynptr_skb_slice_rdwr_buffer", SETUP_SKB_PROG_NONLINEAR}, {"test_dynptr_skb_meta_data", SETUP_SKB_PROG}, {"test_dynptr_skb_meta_flags", SETUP_SKB_PROG}, {"test_adjust", SETUP_SYSCALL_SLEEP}, diff --git a/tools/testing/selftests/bpf/prog_tests/fentry_kprobe_shared.c b/tools/testing/selftests/bpf/prog_tests/fentry_kprobe_shared.c new file mode 100644 index 000000000000..751aa49d6106 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/fentry_kprobe_shared.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "fentry_kprobe_shared.skel.h" + +static void trigger(struct fentry_kprobe_shared *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + int err; + + err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.fentry1), &topts); + ASSERT_OK(err, "test_run"); +} + +/* + * With a kprobe next to the trampoline, the ftrace site of the function + * reaches the trampoline through the list of ftrace callbacks, and so does + * the site of a trampoline that is modified to add or remove a program. + * With the fexit program, the trampoline is entered by a jmp. + */ +void test_fentry_kprobe_shared(void) +{ + struct fentry_kprobe_shared *skel; + + skel = fentry_kprobe_shared__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + skel->bss->pid = getpid(); + + skel->links.fentry1 = bpf_program__attach(skel->progs.fentry1); + if (!ASSERT_OK_PTR(skel->links.fentry1, "attach_fentry1")) + goto out; + skel->links.kprobe1 = bpf_program__attach(skel->progs.kprobe1); + if (!ASSERT_OK_PTR(skel->links.kprobe1, "attach_kprobe1")) + goto out; + trigger(skel); + ASSERT_EQ(skel->bss->fentry1_hits, 1, "fentry1_hits"); + ASSERT_EQ(skel->bss->kprobe_hits, 1, "kprobe_hits"); + + skel->links.fentry2 = bpf_program__attach(skel->progs.fentry2); + if (!ASSERT_OK_PTR(skel->links.fentry2, "attach_fentry2")) + goto out; + trigger(skel); + ASSERT_EQ(skel->bss->fentry1_hits, 2, "fentry1_hits"); + ASSERT_EQ(skel->bss->fentry2_hits, 1, "fentry2_hits"); + ASSERT_EQ(skel->bss->kprobe_hits, 2, "kprobe_hits"); + + skel->links.fexit1 = bpf_program__attach(skel->progs.fexit1); + if (!ASSERT_OK_PTR(skel->links.fexit1, "attach_fexit1")) + goto out; + trigger(skel); + ASSERT_EQ(skel->bss->fentry1_hits, 3, "fentry1_hits"); + ASSERT_EQ(skel->bss->fentry2_hits, 2, "fentry2_hits"); + ASSERT_EQ(skel->bss->fexit_hits, 1, "fexit_hits"); + ASSERT_EQ(skel->bss->kprobe_hits, 3, "kprobe_hits"); + + bpf_link__destroy(skel->links.kprobe1); + skel->links.kprobe1 = NULL; + trigger(skel); + ASSERT_EQ(skel->bss->fentry1_hits, 4, "fentry1_hits"); + ASSERT_EQ(skel->bss->fentry2_hits, 3, "fentry2_hits"); + ASSERT_EQ(skel->bss->fexit_hits, 2, "fexit_hits"); + ASSERT_EQ(skel->bss->kprobe_hits, 3, "kprobe_hits"); +out: + fentry_kprobe_shared__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/freplace_args.c b/tools/testing/selftests/bpf/prog_tests/freplace_args.c new file mode 100644 index 000000000000..d1930af23fa0 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/freplace_args.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "freplace_args.skel.h" +#include "freplace_args_tgt.skel.h" + +/* + * An extension must not expect more of its arguments than the callers of the + * function that it replaces provide. The progs are loaded and attached to each + * other. None of them runs. + */ +static const struct test { + const char *name; + const char *ext; /* prog of freplace_args.c */ + const char *tgt; /* prog of freplace_args_tgt.c */ + const char *func; /* function of tgt that ext is loaded for */ + const char *func2; /* function of tgt that ext is attached to */ + int bad_reg; /* argument that is refused, 0: none */ +} tests[] = { + { "same_nonnull", "ext_mem_nonnull", "tc_prog", "mem_nonnull" }, + { "same_trusted", "ext_task_trusted", "tc_prog", "task_trusted" }, + { "smaller_struct", "ext_mem_smaller", "tc_prog", "mem_smaller" }, + { "larger_struct", "ext_mem_larger", "tc_prog", "mem_larger", NULL, 1 }, + { "nullable_for_nonnull", "ext_mem", "tc_prog", "mem_nonnull" }, + { "nonnull_for_nullable", "ext_mem_nonnull", "tc_prog", "mem", NULL, 2 }, + { "trusted_for_mem", "ext_task_trusted", "tc_prog", "task_mem", NULL, 1 }, + { "arg_ctx_for_ctx_type", "ext_skb_ctx", "tc_prog", "skb_ctx" }, + { "mem_for_arg_ctx", "ext_tp_mem", "tp_prog", "tagged_ctx", NULL, 1 }, + { "attach_nullable_to_nonnull", "ext_mem", "tc_prog", "mem", "mem_nonnull" }, + { "attach_nonnull_to_nullable", "ext_mem_nonnull", "tc_prog", "mem_nonnull", "mem", 2 }, +}; + +static void run_test(struct freplace_args_tgt *tgt_skel, const struct test *t) +{ + struct bpf_program *ext, *tgt; + struct freplace_args *skel; + char log[4096], msg[128]; + struct bpf_link *link; + int err; + + skel = freplace_args__open(); + if (!ASSERT_OK_PTR(skel, "open")) + return; + + ext = bpf_object__find_program_by_name(skel->obj, t->ext); + tgt = bpf_object__find_program_by_name(tgt_skel->obj, t->tgt); + if (!ASSERT_OK_PTR(ext, "ext") || !ASSERT_OK_PTR(tgt, "tgt")) + goto out; + + log[0] = '\0'; + bpf_program__set_autoload(ext, true); + /* libbpf prints the log of a load that fails and must not */ + if (t->bad_reg && !t->func2) + bpf_program__set_log_buf(ext, log, sizeof(log)); + err = bpf_program__set_attach_target(ext, bpf_program__fd(tgt), t->func); + if (!ASSERT_OK(err, "set_attach_target")) + goto out; + + err = freplace_args__load(skel); + if (t->bad_reg && !t->func2) { + snprintf(msg, sizeof(msg), + "Extension program's argument in R%d is incompatible with original", + t->bad_reg); + if (ASSERT_ERR(err, "load")) + ASSERT_HAS_SUBSTR(log, msg, "log"); + goto out; + } + if (!ASSERT_OK(err, "load") || !t->func2) + goto out; + + link = bpf_program__attach_freplace(ext, bpf_program__fd(tgt), t->func2); + ASSERT_EQ(libbpf_get_error(link), t->bad_reg ? -EINVAL : 0, "attach"); + bpf_link__destroy(link); +out: + freplace_args__destroy(skel); +} + +void test_freplace_args(void) +{ + struct freplace_args_tgt *tgt_skel; + int i; + + tgt_skel = freplace_args_tgt__open_and_load(); + if (!ASSERT_OK_PTR(tgt_skel, "tgt_open_and_load")) + return; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (test__start_subtest(tests[i].name)) + run_test(tgt_skel, &tests[i]); + } + + freplace_args_tgt__destroy(tgt_skel); +} \ No newline at end of file diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_far.c b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c index 8d0ff30bd068..b6d262911a3e 100644 --- a/tools/testing/selftests/bpf/prog_tests/may_goto_far.c +++ b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c @@ -34,8 +34,10 @@ static struct bpf_insn *fill(struct bpf_insn *insn, int cnt, int reg) * The prog returns 0 when may_goto jumps to its target and 2 when * it lands in one of 'r0 = r8' areas. There are no constants between * may_goto and its target to keep the distance when constants are blinded. + * + * With 'subprog' all of that is a subprog. */ -static void test_far(int off) +static void test_far(int off, bool subprog) { LIBBPF_OPTS(bpf_prog_load_opts, opts, .prog_flags = BPF_F_SLEEPABLE); LIBBPF_OPTS(bpf_test_run_opts, topts); @@ -50,6 +52,10 @@ static void test_far(int off) return; insn = insns; + if (subprog) { + *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 1); + *insn++ = BPF_EXIT_INSN(); + } *insn++ = BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns); *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_0); *insn++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_6, 63); @@ -82,11 +88,19 @@ static void test_far(int off) void test_may_goto_far(void) { if (test__start_subtest("32762")) - test_far(32762); + test_far(32762, false); if (test__start_subtest("32767")) - test_far(32767); + test_far(32767, false); if (test__start_subtest("-32767")) - test_far(-32767); + test_far(-32767, false); if (test__start_subtest("-32768")) - test_far(-32768); + test_far(-32768, false); + if (test__start_subtest("subprog 1")) + test_far(1, true); + if (test__start_subtest("subprog 32767")) + test_far(32767, true); + if (test__start_subtest("subprog -6")) + test_far(-6, true); + if (test__start_subtest("subprog -32768")) + test_far(-32768, true); } diff --git a/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c b/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c new file mode 100644 index 000000000000..3da3f33a2898 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include +#include +#include +#include "timed_may_goto.skel.h" +#include "timed_may_goto_user.skel.h" +#include "timed_may_goto_ext.skel.h" + +#define TRIES 4 + +static int run_tc(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + ); + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &opts), "test_run")) + return -1; + return opts.retval; +} + +static int run_syscall(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &opts), "test_run")) + return -1; + return opts.retval; +} + +static void wait_for(int *val, int expected, const char *name) +{ + int i; + + for (i = 0; i < 500 && *(volatile int *)val != expected; i++) + usleep(10000); + ASSERT_EQ(*val, expected, name); +} + +static void test_async(struct timed_may_goto_user *skel) +{ + ASSERT_EQ(run_syscall(skel->progs.async_after_timeout), 0, "async_after_timeout"); + wait_for(&skel->bss->timer_cnt, 1000, "timer_cnt"); + wait_for(&skel->bss->wq_cnt, 1000, "wq_cnt"); + wait_for(&skel->bss->tw_cnt, 1000, "tw_cnt"); + wait_for(&skel->bss->rcu_cnt, 1000, "rcu_cnt"); + wait_for(&skel->bss->rcu_trace_cnt, 1000, "rcu_trace_cnt"); +} + +static void test_timer_on_top(struct timed_may_goto_user *skel) +{ + int ms = run_syscall(skel->progs.spin_under_timer); + int ticks = skel->bss->ticks; + + /* stop the timer */ + skel->bss->ticks = 100; + ASSERT_GE(ms, 200, "ms"); + ASSERT_LE(ms, 500, "ms"); + /* control: the callback interrupted the prog, and its loop went on to the end */ + ASSERT_GE(ticks, 5, "ticks"); + ASSERT_LT(ticks, 100, "ticks"); +} + +static void test_freplace(struct timed_may_goto_user *skel) +{ + struct timed_may_goto_ext *ext; + + ASSERT_EQ(run_tc(skel->progs.freplace_target), 1, "not replaced"); + + ext = timed_may_goto_ext__open(); + if (!ASSERT_OK_PTR(ext, "ext_open")) + return; + bpf_program__set_attach_target(ext->progs.replacement, + bpf_program__fd(skel->progs.freplace_target), + "replaced_global"); + if (!ASSERT_OK(timed_may_goto_ext__load(ext), "ext_load") || + !ASSERT_OK(timed_may_goto_ext__attach(ext), "ext_attach")) + goto out; + ASSERT_EQ(run_tc(skel->progs.freplace_target), TRIES, "replaced"); +out: + timed_may_goto_ext__destroy(ext); + ASSERT_EQ(run_tc(skel->progs.freplace_target), 1, "not replaced"); +} + +/* What a prog leaves behind when its time is up doesn't matter for the next one on the cpu. */ +static void test_tail_call_and_throw(struct timed_may_goto_user *skel) +{ + int key = 0, fd = bpf_program__fd(skel->progs.tail_callee); + int map_fd = bpf_map__fd(skel->maps.jmp_table); + + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, &fd, 0), "update")) + return; + + ASSERT_EQ(run_tc(skel->progs.normal_prog), 1, "normal_prog"); + ASSERT_EQ(run_tc(skel->progs.normal_prog), 1, "normal_prog"); + + ASSERT_EQ(run_tc(skel->progs.tail_call_in_subprog), 1, "tail_call_in_subprog"); + ASSERT_EQ(skel->bss->tail_callee_spun, 1, "tail_callee_spun"); + + ASSERT_EQ(run_tc(skel->progs.tail_call_after_timeout), 0, "tail_call_after_timeout"); + ASSERT_EQ(skel->bss->tail_callee_spun, 1, "tail_callee_spun"); + + ASSERT_EQ(run_tc(skel->progs.throw_after_timeout), 7, "throw_after_timeout"); + ASSERT_EQ(run_tc(skel->progs.throw_after_timeout), 7, "throw_after_timeout"); +} + +static void test_user_ringbuf(struct timed_may_goto_user *skel) +{ + struct user_ring_buffer *rb; + void *sample; + int i; + + rb = user_ring_buffer__new(bpf_map__fd(skel->maps.user_ringbuf), NULL); + if (!ASSERT_OK_PTR(rb, "user_ring_buffer__new")) + return; + for (i = 0; i < TRIES; i++) { + sample = user_ring_buffer__reserve(rb, 8); + if (!ASSERT_OK_PTR(sample, "reserve")) + goto out; + user_ring_buffer__submit(rb, sample); + } + /* The helper takes a sample out of the buffer for each call: all calls have to run. */ + ASSERT_EQ(run_syscall(skel->progs.drain), TRIES * 10 + 1, "drain"); +out: + user_ring_buffer__free(rb); +} + +static void test_priv_stack(struct timed_may_goto_user *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int fd = bpf_program__fd(skel->progs.priv_stack); + struct bpf_link *link; + + link = bpf_program__attach(skel->progs.priv_stack); + if (!ASSERT_OK_PTR(link, "attach")) + return; + ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "test_run"); + ASSERT_EQ(skel->bss->priv_stack_spun, 1, "priv_stack_spun"); + ASSERT_EQ(skel->bss->priv_stack_cb_spun, 0, "priv_stack_cb_spun"); + bpf_link__destroy(link); +} + +/* + * Serial: most progs spin for 250 ms in an RCU read-side critical section, one after the other. + * That delays what other tests wait for. In parallel mode "mmap" gives up its wait for the end of + * a map. + */ +void serial_test_timed_may_goto(void) +{ + struct timed_may_goto_user *skel; + cpu_set_t old, new; + + /* The interpreter counts in each frame. */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + RUN_TESTS(timed_may_goto); + +#if !defined(__x86_64__) && !defined(__aarch64__) + return; +#endif + skel = timed_may_goto_user__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + /* Count and timestamp are per-cpu: all progs of a subtest run on one cpu. */ + if (!ASSERT_OK(sched_getaffinity(0, sizeof(old), &old), "getaffinity")) + goto out; + CPU_ZERO(&new); + CPU_SET(sched_getcpu(), &new); + if (!ASSERT_OK(sched_setaffinity(0, sizeof(new), &new), "setaffinity")) + goto out; + + if (test__start_subtest("async")) + test_async(skel); + if (test__start_subtest("timer_on_top")) + test_timer_on_top(skel); + if (test__start_subtest("freplace")) + test_freplace(skel); + if (test__start_subtest("tail_call_and_throw")) + test_tail_call_and_throw(skel); + if (test__start_subtest("user_ringbuf")) + test_user_ringbuf(skel); + if (test__start_subtest("priv_stack")) + test_priv_stack(skel); + + sched_setaffinity(0, sizeof(old), &old); +out: + timed_may_goto_user__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/arena_htab.c b/tools/testing/selftests/bpf/progs/arena_htab.c index 81eaa94afeb0..80cc14bb1c0f 100644 --- a/tools/testing/selftests/bpf/progs/arena_htab.c +++ b/tools/testing/selftests/bpf/progs/arena_htab.c @@ -34,8 +34,6 @@ int arena_htab_llvm(void *ctx) cast_kern(htab); htab_init(htab); - cast_kern(arr); - /* first run. No old elems in the table */ for (i = zero; i < 100000 && can_loop; i++) { htab_update_elem(htab, i, i); diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index c2247b38c849..3770ffdeaaaa 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -1277,6 +1277,270 @@ int skb_invalid_data_slice4(struct __sk_buff *skb) return SK_PASS; } +/* + * bpf_dynptr_slice() and bpf_dynptr_slice_rdwr() return the buffer when the data + * is not contiguous. For the verifier an access through the slice is not an access + * to the stack. Thus the buffer must not hold data that the verifier tracks: a + * store to the buffer invalidates the slice. fp-32: the dynptr, r7: the slice. + */ +#define SLICE_BUF_DYNPTR(new) \ + "r2 = 0;" \ + "r3 = r10;" \ + "r3 += -32;" \ + "call %[" #new "];" + +/* r3 is the buffer */ +#define SLICE_BUF_CALL(size) \ + "r1 = r10;" \ + "r1 += -32;" \ + "r2 = 0;" \ + "r4 = " #size ";" \ + "call %[slice];" + +#define SLICE_BUF_CHECK \ + "if r0 == 0 goto 9f;" \ + "r7 = r0;" + +#define SLICE_BUF_AT(off, size) \ + "r3 = r10;" \ + "r3 += -" #off ";" \ + SLICE_BUF_CALL(size) \ + SLICE_BUF_CHECK + +#define SLICE_BUF(off, size) SLICE_BUF_DYNPTR(bpf_dynptr_from_skb) SLICE_BUF_AT(off, size) + +struct file {}; +extern int bpf_dynptr_from_file(struct file *, __u32, struct bpf_dynptr *) __weak __ksym; + +/* clang makes no BTF for a kfunc that only asm calls */ +void __kfunc_btf_root(void) +{ + bpf_dynptr_from_file(0, 0, 0); +} + +/* Each prog ends with a load through the slice */ +#define SLICE_BUF_PROG(sec, kfunc, name, expect, ...) \ +SEC(sec) \ +expect \ +__naked int name(void) \ +{ \ + asm volatile (__VA_ARGS__ \ + "r0 = *(u8 *)(r7 + 0);" \ +"9:" \ + "r0 = 0;" \ + "exit;" \ + : \ + : __imm(bpf_dynptr_from_file), \ + __imm(bpf_dynptr_from_skb), \ + __imm(bpf_dynptr_from_skb_meta), \ + __imm(bpf_dynptr_from_xdp), \ + __imm(bpf_get_prandom_u32), \ + __imm(bpf_probe_read_kernel), \ + __imm_const(slice, kfunc) \ + : __clobber_all); \ +} +#define SLICE_BUF_TEST(...) SLICE_BUF_PROG("?tc", bpf_dynptr_slice_rdwr, __VA_ARGS__) + +/* Not in the line of SLICE_BUF_TEST(): the log of the verifier has a copy of that line */ +#define SLICE_BUF_INVALID __failure __msg("R7 invalid mem access 'scalar'") +#define SLICE_BUF_POINTER __failure __msg("invalid size of register spill") + +SLICE_BUF_TEST(slice_buf_first_byte, SLICE_BUF_INVALID, + SLICE_BUF(12, 4) + "*(u8 *)(r10 - 12) = 0;") + +SLICE_BUF_TEST(slice_buf_last_byte, SLICE_BUF_INVALID, + SLICE_BUF(12, 4) + "*(u8 *)(r10 - 9) = 0;") + +/* the store is at fp-16 or at fp-8 */ +SLICE_BUF_TEST(slice_buf_store_var_off, SLICE_BUF_INVALID, + SLICE_BUF(8, 8) + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r1 = r10;" + "r1 += -16;" + "r1 += r0;" + "*(u64 *)(r1 + 0) = 0;") + +/* the buffer is at fp-16 or at fp-8 */ +SLICE_BUF_TEST(slice_buf_var_off, SLICE_BUF_INVALID, + SLICE_BUF_DYNPTR(bpf_dynptr_from_skb) + "call %[bpf_get_prandom_u32];" + "r0 &= 8;" + "r3 = r10;" + "r3 += -16;" + "r3 += r0;" + SLICE_BUF_CALL(8) + SLICE_BUF_CHECK + "*(u64 *)(r10 - 8) = 0;") + +/* the slice is a pointer when it is stored to its buffer, and invalid after that */ +SLICE_BUF_TEST(slice_buf_narrow_spill, SLICE_BUF_POINTER, + SLICE_BUF(8, 8) + "*(u32 *)(r10 - 8) = r7;") + +/* the write to the buffer is the spill of the slice, not yet checked for NULL */ +SLICE_BUF_TEST(slice_buf_spilled_or_null, SLICE_BUF_INVALID, + SLICE_BUF_DYNPTR(bpf_dynptr_from_skb) + "r3 = r10;" + "r3 += -8;" + SLICE_BUF_CALL(8) + "*(u64 *)(r10 - 8) = r0;" + "r0 = *(u64 *)(r10 - 8);" + SLICE_BUF_CHECK) + +/* a slice that may be the buffer is the buffer of the next slice */ +SLICE_BUF_TEST(slice_buf_is_a_slice, SLICE_BUF_INVALID, + SLICE_BUF(8, 8) + "r3 = r7;" + SLICE_BUF_CALL(8) + SLICE_BUF_CHECK + "*(u64 *)(r10 - 8) = 0;") + +/* The paths differ only in the buffer. The safe one is first, the other must not be pruned. */ +SLICE_BUF_TEST(slice_buf_pruning, SLICE_BUF_INVALID __flag(BPF_F_TEST_STATE_FREQ), + SLICE_BUF_DYNPTR(bpf_dynptr_from_skb) + "call %[bpf_get_prandom_u32];" + "if r0 == 0 goto 1f;" + SLICE_BUF_AT(16, 8) + "goto 2f;" +"1:" + SLICE_BUF_AT(8, 8) +"2:" + "*(u64 *)(r10 - 8) = 0;") + +#define SLICE_BUF_FUNC(name, ...) \ +static __used __naked void name(void) \ +{ \ + asm volatile (__VA_ARGS__ \ + "exit;" \ + : \ + : __imm(bpf_dynptr_slice) \ + : __clobber_all); \ +} + +/* r1 is the dynptr of the caller. The buffer of the read-only slice is gone after the return. */ +SLICE_BUF_FUNC(slice_buf_in_callee, + "r2 = 0;" + "r3 = r10;" + "r3 += -8;" + "r4 = 8;" + "call %[bpf_dynptr_slice];") + +SLICE_BUF_TEST(slice_buf_frame_returned, SLICE_BUF_INVALID, + SLICE_BUF_DYNPTR(bpf_dynptr_from_skb) + "r1 = r10;" + "r1 += -32;" + "call slice_buf_in_callee;" + SLICE_BUF_CHECK) + +/* stores to fp-12 of this frame and to r1, a pointer to the stack of the caller */ +SLICE_BUF_FUNC(slice_buf_stores, + "*(u32 *)(r10 - 12) = 0;" + "*(u32 *)(r1 + 0) = 0;" + "r0 = 0;") + +SLICE_BUF_TEST(slice_buf_written_by_callee, SLICE_BUF_INVALID, + SLICE_BUF(12, 4) + "r1 = r10;" + "r1 += -12;" + "call slice_buf_stores;") + +/* r1 is the dynptr and r2 the buffer, both in the frame of the caller */ +SLICE_BUF_FUNC(slice_buf_of_caller, + "r3 = r2;" + "r2 = 0;" + "r4 = 4;" + "call %[bpf_dynptr_slice];") + +/* stores next to the buffer or to another frame, a return, calls that fill the buffer */ +SLICE_BUF_TEST(slice_buf_stays_valid, __success, + SLICE_BUF(12, 4) + "r8 = r7;" + "*(u8 *)(r10 - 13) = 0;" + "*(u8 *)(r10 - 8) = 0;" + "r1 = r10;" + "r1 += -16;" + "call slice_buf_stores;" + "r1 = r10;" + "r1 += -12;" + "r2 = 4;" + "r3 = 0;" + "call %[bpf_probe_read_kernel];" + "r1 = r10;" + "r1 += -32;" + "r2 = r10;" + "r2 += -12;" + "call slice_buf_of_caller;" + SLICE_BUF_CHECK + "*(u8 *)(r8 + 0) = 0;") + +/* the slice of a file dynptr is read-only, and it is always the buffer */ +SLICE_BUF_PROG("?lsm.s/file_open", bpf_dynptr_slice, slice_buf_file, SLICE_BUF_INVALID, + "r1 = *(u64 *)(r1 + 0);" + SLICE_BUF_DYNPTR(bpf_dynptr_from_file) + SLICE_BUF_AT(8, 8) + "*(u64 *)(r10 - 8) = 0;") + +SLICE_BUF_PROG("?xdp", bpf_dynptr_slice_rdwr, slice_buf_xdp, SLICE_BUF_INVALID, + SLICE_BUF_DYNPTR(bpf_dynptr_from_xdp) + SLICE_BUF_AT(8, 8) + "*(u64 *)(r10 - 8) = 0;") + +/* the metadata is contiguous: the slice is never the buffer */ +SLICE_BUF_TEST(slice_buf_skb_meta, __success, + SLICE_BUF_DYNPTR(bpf_dynptr_from_skb_meta) + SLICE_BUF_AT(8, 8) + "*(u64 *)(r10 - 8) = 0;") + +extern void bpf_local_irq_save(unsigned long *) __weak __ksym; + +/* a dynptr, an iterator or irq flags are made in the buffer */ +#define SLICE_BUF_SPECIAL(name, type, make) \ +SEC("?tc") \ +__failure __msg("invalid mem access 'scalar'") \ +int name(struct __sk_buff *skb) \ +{ \ + struct bpf_dynptr ptr; \ + type buf; \ + __u8 *p; \ + \ + bpf_dynptr_from_skb(skb, 0, &ptr); \ + p = bpf_dynptr_slice_rdwr(&ptr, 0, &buf, sizeof(buf)); \ + if (!p) \ + return 0; \ + make; \ + return *p; \ +} + +SLICE_BUF_SPECIAL(slice_buf_dynptr, struct bpf_dynptr, bpf_dynptr_from_skb(skb, 0, &buf)) +SLICE_BUF_SPECIAL(slice_buf_iter, struct bpf_iter_num, bpf_iter_num_new(&buf, 0, 1)) +SLICE_BUF_SPECIAL(slice_buf_irq_flag, unsigned long, bpf_local_irq_save(&buf)) + +/* The type of a dynptr argument of a global function is not known: the slice may be the buffer. */ +__noinline int slice_buf_global(struct bpf_dynptr *ptr) +{ + __u64 buf; + __u8 *p; + + p = bpf_dynptr_slice_rdwr(ptr, 0, &buf, sizeof(buf)); + if (!p) + return 0; + *(volatile __u64 *)&buf = 0; + return *p; +} + +SEC("?tc") +__failure __msg("invalid mem access 'scalar'") +int slice_buf_dynptr_arg(struct __sk_buff *skb) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb(skb, 0, &ptr); + return slice_buf_global(&ptr); +} + /* Read-only skb data slice is invalidated on write to skb metadata */ SEC("?tc") __failure __msg("invalid mem access 'scalar'") diff --git a/tools/testing/selftests/bpf/progs/dynptr_success.c b/tools/testing/selftests/bpf/progs/dynptr_success.c index b668ebd61fc7..7949dd538569 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_success.c +++ b/tools/testing/selftests/bpf/progs/dynptr_success.c @@ -231,6 +231,40 @@ int test_dynptr_skb_slice_non_linear(struct __sk_buff *skb) return 1; } +/* + * The tcp header is not in the linear part: the slice is the buffer, and a store + * through the slice is seen in the buffer. As in the documentation, the stores + * come before the compare with the buffer: after it a compiler may use one + * pointer in place of the other. + */ +SEC("?tc") +int test_dynptr_skb_slice_rdwr_buffer(struct __sk_buff *skb) +{ + struct tcphdr buffer = {}, *tcp; + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb(skb, 0, &ptr); + tcp = bpf_dynptr_slice_rdwr(&ptr, TEST_SKB_LINEAR_SIZE, &buffer, sizeof(buffer)); + if (!tcp) { + err = 1; + return 1; + } + + tcp->source = 0x1234; + if (buffer.source != 0x1234) { + err = 2; + return 1; + } + /* a read of the buffer does not invalidate the slice */ + tcp->dest = 0x5678; + if (tcp != &buffer) + err = 3; + else if (bpf_dynptr_write(&ptr, TEST_SKB_LINEAR_SIZE, &buffer, sizeof(buffer), 0)) + err = 4; + + return 1; +} + SEC("?tc") int test_dynptr_skb_meta_data(struct __sk_buff *skb) { @@ -759,6 +793,96 @@ int test_dynptr_copy_xdp(struct xdp_md *xdp) return XDP_DROP; } +static __always_inline int fill_xdp(struct bpf_dynptr *ptr, __u32 start, __u32 n) +{ + __u8 b; + int i; + + bpf_for(i, 0, n) { + b = (start + i) % 251; + if (bpf_dynptr_write(ptr, start + i, &b, 1, 0)) + return 1; + } + return 0; +} + +static __always_inline int check_xdp(struct bpf_dynptr *ptr, __u32 at, __u32 n, __u32 from) +{ + __u8 b; + int i; + + bpf_for(i, 0, n) { + if (bpf_dynptr_read(&b, 1, ptr, at + i, 0)) + return 1; + if (b != (__u8)((from + i) % 251)) + return 1; + } + return 0; +} + +SEC("?xdp") +int test_dynptr_copy_xdp_overlap(struct xdp_md *xdp) +{ + void *data_end = (void *)(long)xdp->data_end; + void *data = (void *)(long)xdp->data; + __u32 off = data_end - data - 1000; + struct bpf_dynptr ptr; + + bpf_dynptr_from_xdp(xdp, 0, &ptr); + if (fill_xdp(&ptr, off, 1300)) { + err = 1; + return XDP_DROP; + } + err = bpf_dynptr_copy(&ptr, off + 100, &ptr, off, 1200); + if (err) + return XDP_DROP; + if (check_xdp(&ptr, off + 100, 1200, off)) + err = 2; + return XDP_DROP; +} + +SEC("?xdp") +int test_dynptr_copy_xdp_overlap_src_linear(struct xdp_md *xdp) +{ + void *data_end = (void *)(long)xdp->data_end; + void *data = (void *)(long)xdp->data; + __u32 lin = data_end - data; + struct bpf_dynptr ptr; + + bpf_dynptr_from_xdp(xdp, 0, &ptr); + if (fill_xdp(&ptr, lin - 64, 118)) { + err = 1; + return XDP_DROP; + } + err = bpf_dynptr_copy(&ptr, lin - 10, &ptr, lin - 64, 64); + if (err) + return XDP_DROP; + if (check_xdp(&ptr, lin - 10, 64, lin - 64)) + err = 2; + return XDP_DROP; +} + +SEC("?xdp") +int test_dynptr_copy_xdp_overlap_dst_frag(struct xdp_md *xdp) +{ + void *data_end = (void *)(long)xdp->data_end; + void *data = (void *)(long)xdp->data; + __u32 lin = data_end - data; + struct bpf_dynptr ptr; + + bpf_dynptr_from_xdp(xdp, 0, &ptr); + if (fill_xdp(&ptr, lin - 10, 84)) { + err = 1; + return XDP_DROP; + } + err = bpf_dynptr_copy(&ptr, lin + 10, &ptr, lin - 10, 64); + if (err) + return XDP_DROP; + if (check_xdp(&ptr, lin + 10, 64, lin - 10)) + err = 2; + return XDP_DROP; +} + char memset_zero_data[] = "data to be zeroed"; SEC("?tp/syscalls/sys_enter_nanosleep") diff --git a/tools/testing/selftests/bpf/progs/fentry_kprobe_shared.c b/tools/testing/selftests/bpf/progs/fentry_kprobe_shared.c new file mode 100644 index 000000000000..3d16773aada8 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/fentry_kprobe_shared.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include +#include + +char _license[] SEC("license") = "GPL"; + +int pid; +int fentry1_hits; +int fentry2_hits; +int fexit_hits; +int kprobe_hits; + +static bool from_test(int a) +{ + return (bpf_get_current_pid_tgid() >> 32) == pid && a == 1; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(fentry1, int a) +{ + if (from_test(a)) + fentry1_hits++; + return 0; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(fentry2, int a) +{ + if (from_test(a)) + fentry2_hits++; + return 0; +} + +SEC("fexit/bpf_fentry_test1") +int BPF_PROG(fexit1, int a, int ret) +{ + if (from_test(a) && ret == 2) + fexit_hits++; + return 0; +} + +SEC("kprobe/bpf_fentry_test1") +int BPF_KPROBE(kprobe1, int a) +{ + if (from_test(a)) + kprobe_hits++; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/freplace_args.c b/tools/testing/selftests/bpf/progs/freplace_args.c new file mode 100644 index 000000000000..6e8f16bd5d8e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_args.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +/* In freplace_args_tgt.c each of them has 16 bytes. */ +struct buf { __u64 v[2]; }; +struct larger { __u64 v[4]; }; +struct smaller { __u64 v[1]; }; +struct tp_ctx { __u64 v[2]; }; + +SEC("?freplace") +int ext_mem(int a, struct buf *p) +{ + return 0; +} + +SEC("?freplace") +int ext_mem_nonnull(int a, struct buf *p __arg_nonnull) +{ + return 0; +} + +SEC("?freplace") +int ext_mem_larger(struct larger *p) +{ + return 0; +} + +SEC("?freplace") +int ext_mem_smaller(struct smaller *p) +{ + return 0; +} + +SEC("?freplace") +int ext_task_trusted(struct task_struct *t __arg_trusted) +{ + return 0; +} + +SEC("?freplace") +int ext_skb_ctx(struct __sk_buff *skb __arg_ctx) +{ + return 0; +} + +SEC("?freplace") +int ext_tp_mem(struct tp_ctx *p) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/freplace_args_tgt.c b/tools/testing/selftests/bpf/progs/freplace_args_tgt.c new file mode 100644 index 000000000000..4f1d1d3d2891 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_args_tgt.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +/* In freplace_args.c 'larger' has 32 bytes and 'smaller' has 8 bytes. */ +struct buf { __u64 v[2]; }; +struct larger { __u64 v[2]; }; +struct smaller { __u64 v[2]; }; +struct tp_ctx { __u64 v[2]; }; + +__weak int mem(int a, struct buf *p) +{ + return 0; +} + +__weak int mem_nonnull(int a, struct buf *p __arg_nonnull) +{ + return 0; +} + +__weak int mem_larger(struct larger *p) +{ + return 0; +} + +__weak int mem_smaller(struct smaller *p) +{ + return 0; +} + +/* No tag: memory of the size of the struct. */ +__weak int task_mem(struct task_struct *t) +{ + return 0; +} + +__weak int task_trusted(struct task_struct *t __arg_trusted) +{ + return 0; +} + +__weak int skb_ctx(struct __sk_buff *skb) +{ + return 0; +} + +SEC("tc") +int tc_prog(struct __sk_buff *skb) +{ + struct smaller smaller = {}; + struct larger larger = {}; + struct buf buf = {}; + + return mem(0, &buf) + mem_nonnull(0, &buf) + mem_larger(&larger) + mem_smaller(&smaller) + + task_mem(NULL) + task_trusted(bpf_get_current_task_btf()) + skb_ctx(skb); +} + +/* In a tracepoint prog the tag makes a context of any struct. */ +__weak int tagged_ctx(struct tp_ctx *ctx __arg_ctx) +{ + return 0; +} + +SEC("tracepoint") +int tp_prog(void *ctx) +{ + return tagged_ctx(ctx); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/timed_may_goto.c b/tools/testing/selftests/bpf/progs/timed_may_goto.c new file mode 100644 index 000000000000..c1c48af808a9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/timed_may_goto.c @@ -0,0 +1,898 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * There is one time slice of may_goto for an invocation of a prog: one call from the kernel into + * the main prog, with the subprogs and sync callbacks that it reaches. + * + * Most tests call a function that spins until may_goto stops it, four times, each in a different + * way. Only the first call can spin. With a time slice for each call all four do. + */ +#include +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +char _license[] SEC("license") = "GPL"; + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && __clang_major__ >= 18 + +#define TRIES 4 + +int zero; + +/* Return 1 if the loop went on until the time was up. */ +static __always_inline int spin(void) +{ + __u64 t0 = bpf_ktime_get_ns(); + + while (can_loop) + ; + return bpf_ktime_get_ns() - t0 > 100 * 1000 * 1000; +} + +static __noinline int spins(void) +{ + return spin(); +} + +static __noinline int counts_to_1000(void) +{ + int i, cnt = zero; + + for (i = zero; i < 1000 && can_loop; i++) + cnt++; + return cnt; +} + +SEC("tc") +__description("timed may_goto: call") +__success __retval(1) +int call_subprog(void *ctx) +{ + int i, n = 0; + + for (i = 0; i < TRIES; i++) + n += spins(); + return n; +} + +SEC("tc") +__description("timed may_goto: a call after the time is up makes no iteration") +__success __retval(0) +int short_loop_after_timeout(void *ctx) +{ + spins(); + return counts_to_1000(); +} + +SEC("tc") +__description("timed may_goto: the loop of the main prog stops when the time is up in a subprog") +__success __retval(1) +int main_loop_after_timeout(void *ctx) +{ + int n = 0; + + while (can_loop && n < 2 * TRIES) + n += spins(); + return n; +} + +static __noinline int passes(int i) +{ + return spins() + i; +} + +SEC("tc") +__description("timed may_goto: call through a function without a loop") +__success __retval(1) +int call_through(void *ctx) +{ + int i, n = 0; + + for (i = 0; i < TRIES; i++) + n += passes(zero); + return n; +} + +__noinline int global_spins(void) +{ + return spin(); +} + +SEC("tc") +__description("timed may_goto: call of a global function") +__success __retval(1) +int call_global(void *ctx) +{ + int i, n = 0; + + for (i = 0; i < TRIES; i++) + n += global_spins(); + return n; +} + +SEC("tc") +__description("timed may_goto: callx") +__success __retval(1) +__naked void callx_subprog(void) +{ + asm volatile ( + "r6 = 0;" + "r7 = 0;" + "1:" + "r3 = %[spins] ll;" + "callx r3;" + "r6 += r0;" + "r7 += 1;" + "if r7 < 4 goto 1b;" + "r0 = r6;" + "exit;" + : + : __imm_addr(spins) + : __clobber_all); +} + +/* With a time slice for each call it gives up, so that the test fails and doesn't lock up. */ +static int loop_cb(__u64 i, int *spun) +{ + *spun += spin(); + return *spun > 2 * TRIES; +} + +SEC("tc") +__description("timed may_goto: bpf_loop() inlined") +__success __retval(1) +int loop_inlined(void *ctx) +{ + int spun = 0; + + bpf_loop(1 << 20, loop_cb, &spun, 0); + return spun; +} + +SEC("tc") +__description("timed may_goto: bpf_loop() not inlined") +__success __retval(1) +int loop_not_inlined(void *ctx) +{ + int spun = 0; + + bpf_loop(1 << 20, loop_cb, &spun, zero); + return spun; +} + +static int count_cb(__u64 i, int *calls) +{ + (*calls)++; + return 0; +} + +/* The time is up: the helper calls the callback one more time. It returns 1 in its first insns. */ +#define LOOP_ENDS(name, flags) \ +SEC("tc") \ +__description("timed may_goto: bpf_loop() " #name " ends when the time is up") \ +__success __retval(10) \ +int loop_ends_##name(void *ctx) \ +{ \ + int calls = 0; \ + \ + spins(); \ + return bpf_loop(1000, count_cb, &calls, flags) * 10 + calls; \ +} + +LOOP_ENDS(inlined, 0) +LOOP_ENDS(not_inlined, zero) + +struct nested { + __u64 t0; + int spun; +}; + +/* Without an end of bpf_loop(): give up, so that the test fails and doesn't lock up. */ +static int inner_cb(__u64 i, struct nested *n) +{ + if (!n->spun) + n->spun = spin(); + return (i & 1023) == 0 && bpf_ktime_get_ns() - n->t0 > 1000 * 1000 * 1000; +} + +static int outer_cb(__u64 i, struct nested *n) +{ + bpf_loop(1 << 23, inner_cb, n, 0); + return bpf_ktime_get_ns() - n->t0 > 1000 * 1000 * 1000; +} + +SEC("tc") +__description("timed may_goto: bpf_loop() in bpf_loop() ends when the time is up") +__success __retval(1) +int loop_in_loop(void *ctx) +{ + struct nested n = { .t0 = bpf_ktime_get_ns() }; + + bpf_loop(1 << 23, outer_cb, &n, 0); + return n.spun && bpf_ktime_get_ns() - n.t0 < 500 * 1000 * 1000; +} + +static __noinline int called_cb(__u64 i, int *calls) +{ + (*calls)++; + return 0; +} + +/* BPF code calls the callback too, so its return value has to be its own. bpf_loop() goes on. */ +SEC("tc") +__description("timed may_goto: bpf_loop() with a callback that is called as well") +__success __retval(45) +int loop_and_call(void *ctx) +{ + int calls = 0; + + spins(); + if (called_cb(zero, &calls)) + return 100; + return bpf_loop(TRIES, called_cb, &calls, 0) * 10 + calls; +} + +static int short_cb(__u64 i, __u64 *t0) +{ + int j; + + for (j = zero; j < 60000 && can_loop; j++) + ; + return (i & 1023) == 0 && bpf_ktime_get_ns() - *t0 > 1000 * 1000 * 1000; +} + +/* + * The main prog has no may_goto. Each call of the callback ends before it reads the clock. The + * iterations of all calls add up. + */ +SEC("tc") +__description("timed may_goto: bpf_loop() the callback makes 60000 iterations") +__success __retval(1) +int loop_short_cb(void *ctx) +{ + __u64 t0 = bpf_ktime_get_ns(); + + bpf_loop(1 << 20, short_cb, &t0, 0); + return bpf_ktime_get_ns() - t0 < 1000 * 1000 * 1000; +} + +/* spins() uses up the time. The loop that calls it ends at its next may_goto. */ +static __noinline int loops_around_spins(void) +{ + int n = zero; + + while (can_loop && n < 20000) { + if (n < 2 * TRIES) + spins(); + n++; + } + return n; +} + +SEC("tc") +__description("timed may_goto: a loop that is on the stack ends when the time is up") +__success __retval(1) +int rest_of_batch(void *ctx) +{ + return loops_around_spins(); +} + +static __noinline int one_iteration(void) +{ + int i, n = zero; + + for (i = zero; i < 1 && can_loop; i++) + n++; + return n; +} + +long calls_ok; + +static int call_one(__u64 i, __u64 *t0) +{ + if (!one_iteration()) + return 1; + calls_ok++; + /* With a time slice for each call: give up. */ + return (i & 1023) == 0 && bpf_ktime_get_ns() - *t0 > 1000 * 1000 * 1000; +} + +static int call_many(__u64 i, __u64 *t0) +{ + long before = calls_ok; + + bpf_loop(1 << 23, call_one, t0, 0); + return calls_ok - before != 1 << 23; +} + +/* + * The iterations of all calls count: the prog reads the clock in each 65535th call. The call that + * finds out that the time is up makes its iteration. + */ +SEC("tc") +__description("timed may_goto: 65535 calls that make one iteration read the clock") +__success __retval(1) +int clock_after_65535_calls(void *ctx) +{ + __u64 t0 = bpf_ktime_get_ns(); + + calls_ok = 0; + bpf_loop(1000, call_many, &t0, 0); + return calls_ok > 0 && calls_ok % 65535 == 0; +} + +#define MAP(name, map_type, flags) \ +struct { \ + __uint(type, map_type); \ + __uint(max_entries, TRIES); \ + __uint(map_flags, flags); \ + __type(key, int); \ + __type(value, int); \ +} name SEC(".maps") + +MAP(array, BPF_MAP_TYPE_ARRAY, 0); +MAP(hash, BPF_MAP_TYPE_HASH, 0); + +/* The first call uses up the time. The second call ends the loop of the helper. */ +static int elem_cb(void *map, int *key, int *val, int *spun) +{ + *spun += spin(); + return 0; +} + +#define FOR_EACH(name) \ +SEC("tc") \ +__description("timed may_goto: bpf_for_each_map_elem() " #name) \ +__success __retval(21) \ +int for_each_##name(void *ctx) \ +{ \ + int i, key, spun = 0; \ + \ + for (i = 0; i < TRIES; i++) { \ + key = i; \ + bpf_map_update_elem(&name, &key, &key, 0); \ + } \ + return bpf_for_each_map_elem(&name, elem_cb, &spun, 0) * 10 + spun; \ +} + +FOR_EACH(array) +FOR_EACH(hash) + +static long vma_cb(struct task_struct *task, struct vm_area_struct *vma, int *spun) +{ + *spun += spin(); + return 0; +} + +SEC("raw_tp") +__description("timed may_goto: bpf_find_vma()") +__success __retval(1) +int find_vma(void *ctx) +{ + struct task_struct *task = bpf_get_current_task_btf(); + __u64 addr = task->mm->arg_start; + int i, spun = 0; + + for (i = 0; i < TRIES; i++) + if (bpf_find_vma(task, addr, vma_cb, &spun, 0)) + return 100; + return spun; +} + +struct tree_node { + struct bpf_rb_node rb; + int key; +}; + +#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) +private(A) struct bpf_spin_lock lock; +private(A) struct bpf_rb_root root __contains(tree_node, rb); + +int less_spun, less_calls; + +static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + long n = zero; + + /* no helpers under the lock */ + while (can_loop) + n++; + less_calls++; + less_spun += n > 1000 * 1000; + return container_of(a, struct tree_node, rb)->key < + container_of(b, struct tree_node, rb)->key; +} + +SEC("tc") +__description("timed may_goto: bpf_rbtree_add()") +__success __retval(1) +int rbtree_add(void *ctx) +{ + struct bpf_rb_node *rb; + struct tree_node *n; + int i; + + less_spun = less_calls = 0; + for (i = 0; i < TRIES; i++) { + n = bpf_obj_new(typeof(*n)); + if (!n) + return 100; + n->key = TRIES - i; + bpf_spin_lock(&lock); + bpf_rbtree_add(&root, &n->rb, less); + bpf_spin_unlock(&lock); + } + for (i = 0; i < TRIES; i++) { + bpf_spin_lock(&lock); + rb = bpf_rbtree_first(&root); + if (rb) + rb = bpf_rbtree_remove(&root, rb); + bpf_spin_unlock(&lock); + if (!rb) + return 101; + bpf_obj_drop(container_of(rb, struct tree_node, rb)); + } + if (less_calls < TRIES - 1) + return 102; + return less_spun; +} + +/* + * All registers are live at the may_goto, so it saves one of them in the stack. They are the same + * after 70000 iterations: the loop reads the clock. + */ +__naked __noinline __used +static long regs_across_may_goto(void) +{ + asm volatile ( + "r0 = 10;" + "r6 = 6;" + "r7 = 7;" + "r8 = 8;" + "r9 = 0;" + "1:" + "may_goto 2f;" + "r9 += 1;" + "if r9 < 70000 goto 1b;" + "2:" + "r0 += r1;" + "r0 += r2;" + "r0 += r3;" + "r0 += r4;" + "r0 += r5;" + "r0 += r6;" + "r0 += r7;" + "r0 += r8;" + "r0 += r9;" + "exit;" + ::: __clobber_all); +} + +SEC("tc") +__description("timed may_goto: registers across may_goto") +__success __retval(70046) +__naked void regs_across_may_goto_main(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call regs_across_may_goto;" + "exit;" + ::: __clobber_all); +} + +/* + * may_goto uses a register that is dead. In live_r0_rN() R0 - RN are live at the may_goto and the + * next one is not. The values come from memory, so the verifier doesn't know them. Return their sum + * and the number of iterations. + */ +__u64 vals[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; +__u64 zero64; + +#define LD0(b) "r0 = *(u64 *)(r" #b " + 0);" +#define LD1(b) LD0(b) "r1 = *(u64 *)(r" #b " + 8);" +#define LD2(b) LD1(b) "r2 = *(u64 *)(r" #b " + 16);" +#define LD3(b) LD2(b) "r3 = *(u64 *)(r" #b " + 24);" +#define LD4(b) LD3(b) "r4 = *(u64 *)(r" #b " + 32);" +#define LD5(b) LD4(b) "r5 = *(u64 *)(r" #b " + 40);" +#define LD6(b) LD5(b) "r6 = *(u64 *)(r" #b " + 48);" +#define LD7(b) LD6(b) "r7 = *(u64 *)(r" #b " + 56);" +#define LD8(b) LD7(b) "r8 = *(u64 *)(r" #b " + 64);" + +#define SUM0 +#define SUM1 SUM0 "r0 += r1;" +#define SUM2 SUM1 "r0 += r2;" +#define SUM3 SUM2 "r0 += r3;" +#define SUM4 SUM3 "r0 += r4;" +#define SUM5 SUM4 "r0 += r5;" +#define SUM6 SUM5 "r0 += r6;" +#define SUM7 SUM6 "r0 += r7;" +#define SUM8 SUM7 "r0 += r8;" + +/* The count of iterations is at r10 - 8. It begins with a number from memory too. */ +#define COUNT_INIT(t) \ + "r" #t " = %[zero64] ll;" \ + "r" #t " = *(u64 *)(r" #t " + 0);" \ + "*(u64 *)(r10 - 8) = r" #t ";" +#define COUNT_AND_LOOP(t) \ + "r" #t " = *(u64 *)(r10 - 8);" \ + "r" #t " += 1;" \ + "*(u64 *)(r10 - 8) = r" #t ";" \ + "if r" #t " < 70000 goto 1b;" +#define ADD_COUNT_AND_EXIT \ + "r1 = *(u64 *)(r10 - 8);" \ + "r0 += r1;" \ + "exit;" \ + : \ + : __imm_addr(vals), __imm_addr(zero64) \ + : __clobber_all + +#define LIVE_REGS(n, next) \ +__naked __noinline __used \ +static long live_r0_r##n(void) \ +{ \ + asm volatile ( \ + COUNT_INIT(next) \ + "r" #n " = %[vals] ll;" \ + LD##n(n) \ + "1:" \ + "may_goto 2f;" \ + COUNT_AND_LOOP(next) \ + "2:" \ + SUM##n \ + ADD_COUNT_AND_EXIT); \ +} + +LIVE_REGS(0, 1) +LIVE_REGS(1, 2) +LIVE_REGS(2, 3) +LIVE_REGS(3, 4) +LIVE_REGS(4, 5) +LIVE_REGS(5, 6) +LIVE_REGS(6, 7) +LIVE_REGS(7, 8) +LIVE_REGS(8, 9) + +/* R2 is read at the top of the loop only: it is live at the may_goto through the back edge. */ +__naked __noinline __used +static long live_back_edge(void) +{ + asm volatile ( + COUNT_INIT(3) + "r2 = %[vals] ll;" + LD2(2) + "1:" + "*(u64 *)(r10 - 16) = r2;" + "may_goto 2f;" + COUNT_AND_LOOP(3) + "2:" + "r0 += r1;" + "r2 = *(u64 *)(r10 - 16);" + "r0 += r2;" + ADD_COUNT_AND_EXIT); +} + +/* R2 is read only where the may_goto jumps to. */ +__naked __noinline __used +static long live_target(void) +{ + asm volatile ( + COUNT_INIT(3) + "r2 = %[vals] ll;" + LD2(2) + "1:" + "may_goto 2f;" + COUNT_AND_LOOP(3) + "goto 3f;" + "2:" + "r0 += r2;" + "3:" + "r0 += r1;" + ADD_COUNT_AND_EXIT); +} + +/* R2 is set in front of the may_goto in each iteration, and read only right behind it. */ +__naked __noinline __used +static long live_fall_through(void) +{ + asm volatile ( + COUNT_INIT(3) + "r1 = %[vals] ll;" + LD1(1) + "*(u64 *)(r10 - 16) = r0;" + "1:" + "r2 = %[vals] ll;" + "r2 = *(u64 *)(r2 + 16);" + "may_goto 2f;" + "*(u64 *)(r10 - 16) = r2;" + COUNT_AND_LOOP(3) + "2:" + "r0 += r1;" + "r2 = *(u64 *)(r10 - 16);" + "r0 += r2;" + ADD_COUNT_AND_EXIT); +} + +#define CALL_AND_ADD(f) "call " #f ";" "r6 += r0;" +#define CALL_ALL_LIVE \ + "r6 = 0;" \ + CALL_AND_ADD(live_r0_r0) \ + CALL_AND_ADD(live_r0_r1) \ + CALL_AND_ADD(live_r0_r2) \ + CALL_AND_ADD(live_r0_r3) \ + CALL_AND_ADD(live_r0_r4) \ + CALL_AND_ADD(live_r0_r5) \ + CALL_AND_ADD(live_r0_r6) \ + CALL_AND_ADD(live_r0_r7) \ + CALL_AND_ADD(live_r0_r8) \ + CALL_AND_ADD(live_back_edge) \ + CALL_AND_ADD(live_target) \ + CALL_AND_ADD(live_fall_through) \ + "r0 = r6;" \ + "exit;" + +/* + * The sums: 1 + 3 + 6 + 10 + 15 + 21 + 28 + 36 + 45 = 165. Then 6, 3 (the may_goto doesn't jump) + * and 6. 12 * 70000 iterations: each function reads the clock. + */ +SEC("tc") +__description("timed may_goto: may_goto takes a register that is dead") +__success __retval(840180) +__naked void dead_reg(void) +{ + asm volatile ( + CALL_ALL_LIVE + ::: __clobber_all); +} + +/* No iteration: 165, then 6, 6 (the may_goto jumps) and 4 (R2 isn't stored). */ +SEC("tc") +__description("timed may_goto: may_goto takes a register that is dead, the time is up") +__success __retval(181) +__naked void dead_reg_time_is_up(void) +{ + asm volatile ( + "call spins_on_may_goto;" + CALL_ALL_LIVE + ::: __clobber_all); +} + +/* + * The subprog uses up the time of the prog and returns. With a time slice for each call the caller + * calls it 65535 times before it reads the clock. + */ +__naked __noinline __used +static unsigned long spins_on_may_goto(void) +{ + asm volatile ( + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* The first may_goto of the function finds that the time is up */ +SEC("tc") +__description("timed may_goto: registers across may_goto when the time is up") +__success __retval(46) +__naked void regs_when_time_is_up(void) +{ + asm volatile ( + "call spins_on_may_goto;" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call regs_across_may_goto;" + "exit;" + ::: __clobber_all); +} + +/* The first insn of the main prog is the target of a jump. It must not set the count again. */ +SEC("socket") +__description("timed may_goto: 9 insns") +__success __retval(1) +__naked void loop_calls_subprog_with_may_goto(void) +{ + asm volatile ( + "1:" + "call spins_on_may_goto;" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +/* + * These subprogs make 60000 iterations in each call. With a read of the clock after 65535 + * iterations of a call none of them reads it, and their caller does it after + * 65535 * 60000 * 60000 iterations. + */ +__naked __noinline __used +static unsigned long loops_60000(void) +{ + asm volatile ( + "r1 = 60000;" + "1:" + "may_goto 2f;" + "r1 += -1;" + "if r1 != 0 goto 1b;" + "2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +__naked __noinline __used +static unsigned long calls_60000_times(void) +{ + asm volatile ( + "r6 = 60000;" + "1:" + "call loops_60000;" + "may_goto 2f;" + "r6 += -1;" + "if r6 != 0 goto 1b;" + "2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("timed may_goto: three levels of loops that end before they read the clock") +__success __retval(1) +__naked void three_levels_of_short_loops(void) +{ + asm volatile ( + "1:" + "call calls_60000_times;" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +/* The callee ends one iteration before it reads the clock. */ +__naked __noinline __used +static unsigned long loops_65534(void) +{ + asm volatile ( + "r1 = 65534;" + "1:" + "may_goto 2f;" + "r1 += -1;" + "if r1 != 0 goto 1b;" + "2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("timed may_goto: the subprog makes 65534 iterations") +__success __retval(1) +__naked void loop_calls_subprog_with_65534_iterations(void) +{ + asm volatile ( + "call %[bpf_ktime_get_ns];" + "r6 = r0;" + "1:" + "call loops_65534;" + "may_goto 2f;" + "goto 1b;" + "2:" + "call %[bpf_ktime_get_ns];" + "r0 -= r6;" + "r1 = r0;" + "r0 = 1;" + "if r1 < 1000000000 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* The subprog returns 16 bytes in R0 and R2. may_goto must not change them. */ +__naked __noinline __used +static unsigned __int128 returns_pair_after_may_goto(void) +{ + asm volatile ( + "r0 = 1;" + "r2 = 2;" + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("timed may_goto: the subprog returns 16 bytes") +__success __retval(3) +__naked void subprog_with_may_goto_returns_pair(void) +{ + asm volatile ( + "call returns_pair_after_may_goto;" + "r0 += r2;" + "exit;" + ::: __clobber_all); +} + +/* Dead code doesn't count: a prog that has no may_goto left gets no insns. */ +__naked __noinline __used +static unsigned long may_goto_is_dead_code(void) +{ + asm volatile ( + "r0 = 0;" + "if r0 == 0 goto 2f;" + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("timed may_goto: may_goto in dead code") +__success __retval(0) +__xlated("0: call pc+1") +__xlated("1: exit") +__xlated("2: r0 = 0") +__xlated("3: exit") +__naked void dead_may_goto(void) +{ + asm volatile ( + "call may_goto_is_dead_code;" + "exit;" + ::: __clobber_all); +} + +static __noinline int throws_after_spin(void) +{ + spin(); + bpf_throw(0); + return 0; +} + +__noinline int exception_cb_loops(u64 cookie) +{ + return counts_to_1000(); +} + +/* The exception callback belongs to the invocation that throws. */ +SEC("tc") +__description("timed may_goto: bpf_throw() the exception callback has a loop") +__exception_cb(exception_cb_loops) +__success __retval(0) +int throw_after_timeout(void *ctx) +{ + return throws_after_spin(); +} + +#else + +SEC("socket") +__description("timed may_goto: not supported dummy test") +__success +int dummy_test(void) +{ + return 0; +} + +#endif diff --git a/tools/testing/selftests/bpf/progs/timed_may_goto_ext.c b/tools/testing/selftests/bpf/progs/timed_may_goto_ext.c new file mode 100644 index 000000000000..969ca58b6eed --- /dev/null +++ b/tools/testing/selftests/bpf/progs/timed_may_goto_ext.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +char _license[] SEC("license") = "GPL"; + +static __noinline int spins(void) +{ + __u64 t0 = bpf_ktime_get_ns(); + + while (can_loop) + ; + return bpf_ktime_get_ns() - t0 > 100 * 1000 * 1000; +} + +/* Each call of an extension prog is an invocation: one of its two calls spins. */ +SEC("freplace") +int replacement(int i) +{ + return spins() + spins() + i; +} diff --git a/tools/testing/selftests/bpf/progs/timed_may_goto_user.c b/tools/testing/selftests/bpf/progs/timed_may_goto_user.c new file mode 100644 index 000000000000..8b85815d45d2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/timed_may_goto_user.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0 +/* The tests of timed_may_goto.c that need help from user space. */ +#include +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +char _license[] SEC("license") = "GPL"; + +#define TRIES 4 +#define CLOCK_MONOTONIC 1 + +int zero; + +/* Return 1 if the loop went on until the time was up. */ +static __always_inline int spin(void) +{ + __u64 t0 = bpf_ktime_get_ns(); + + while (can_loop) + ; + return bpf_ktime_get_ns() - t0 > 100 * 1000 * 1000; +} + +static __noinline int spins(void) +{ + return spin(); +} + +static __noinline int counts_to_1000(void) +{ + int i, cnt = zero; + + for (i = zero; i < 1000 && can_loop; i++) + cnt++; + return cnt; +} + +/* Nothing special: the loops go on, then the time is up one time. */ +SEC("tc") +int normal_prog(void *ctx) +{ + int i, n = 0; + + if (counts_to_1000() != 1000) + return 100; + for (i = 0; i < TRIES; i++) + n += spins(); + return n; +} + +/* + * Async callbacks + */ +struct elem { + struct bpf_timer timer; + struct bpf_wq wq; + struct bpf_task_work tw; + struct bpf_rcu_head rcu; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 2); + __type(key, int); + __type(value, struct elem); +} async SEC(".maps"); + +int timer_cnt, wq_cnt, tw_cnt, rcu_cnt, rcu_trace_cnt; + +static int timer_cb(void *map, int *key, struct elem *e) +{ + timer_cnt = counts_to_1000(); + return 0; +} + +static int wq_cb(void *map, int *key, void *val) +{ + wq_cnt = counts_to_1000(); + return 0; +} + +static int tw_cb(struct bpf_map *map, void *key, void *val) +{ + tw_cnt = counts_to_1000(); + return 0; +} + +static int rcu_cb(struct bpf_map *map, void *key, void *val) +{ + rcu_cnt = counts_to_1000(); + return 0; +} + +static int rcu_trace_cb(struct bpf_map *map, void *key, void *val) +{ + rcu_trace_cnt = counts_to_1000(); + return 0; +} + +/* + * The time of this prog is up when it starts the callbacks. User space keeps the task on the cpu, + * so the callbacks find what it has left. Each call of a callback is an invocation of its own. + */ +SEC("syscall") +int async_after_timeout(void *ctx) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct elem *e, *e1; + int key = 0, key1 = 1; + + e = bpf_map_lookup_elem(&async, &key); + e1 = bpf_map_lookup_elem(&async, &key1); + if (!e || !e1) + return 100; + if (bpf_timer_init(&e->timer, &async, CLOCK_MONOTONIC) || + bpf_timer_set_callback(&e->timer, timer_cb)) + return 101; + if (bpf_wq_init(&e->wq, &async, 0) || bpf_wq_set_callback(&e->wq, wq_cb, 0)) + return 102; + if (!spins()) + return 103; + /* control: the time is up */ + if (counts_to_1000()) + return 109; + if (bpf_timer_start(&e->timer, 0, BPF_F_TIMER_CPU_PIN)) + return 104; + if (bpf_wq_start(&e->wq, 0)) + return 105; + if (bpf_task_work_schedule_resume(task, &e->tw, &async, tw_cb)) + return 106; + if (bpf_call_rcu(&e->rcu, &async, rcu_cb)) + return 107; + if (bpf_call_rcu_tasks_trace(&e1->rcu, &async, rcu_trace_cb)) + return 108; + return 0; +} + +int ticks; + +static int tick_cb(void *map, int *key, struct elem *e) +{ + if (counts_to_1000() != 1000) + ticks = 1000; + if (++ticks < 100) + bpf_timer_start(&e->timer, 20 * 1000 * 1000, BPF_F_TIMER_CPU_PIN); + return 0; +} + +/* + * A callback of the prog interrupts it, on the same cpu, each 20 ms. It has its own time. That + * doesn't give more time to the prog. Return how long the loop went on, in ms. + */ +SEC("syscall") +int spin_under_timer(void *ctx) +{ + struct elem *e; + int key = 1; + __u64 t0; + + e = bpf_map_lookup_elem(&async, &key); + if (!e) + return 0; + if (bpf_timer_init(&e->timer, &async, CLOCK_MONOTONIC) || + bpf_timer_set_callback(&e->timer, tick_cb) || + bpf_timer_start(&e->timer, 20 * 1000 * 1000, BPF_F_TIMER_CPU_PIN)) + return 0; + t0 = bpf_ktime_get_ns(); + while (can_loop) + ; + return (bpf_ktime_get_ns() - t0) / 1000000; +} + +/* + * Extension prog + */ +__noinline int replaced_global(int i) +{ + return spin() + i; +} + +SEC("tc") +int freplace_target(void *ctx) +{ + int i, n = 0; + + for (i = 0; i < TRIES; i++) + n += replaced_global(zero); + return n; +} + +/* + * Tail calls + */ +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} jmp_table SEC(".maps"); + +int tail_callee_spun; + +SEC("tc") +int tail_callee(struct __sk_buff *skb) +{ + int i; + + tail_callee_spun = 0; + for (i = 0; i < TRIES; i++) + tail_callee_spun += spins(); + return 0; +} + +static __noinline int tail_calls(struct __sk_buff *skb) +{ + bpf_tail_call(skb, &jmp_table, 0); + return zero + 100; +} + +/* The target of the tail call is another prog. It returns to this function. */ +SEC("tc") +int tail_call_in_subprog(struct __sk_buff *skb) +{ + int i, n; + + n = tail_calls(skb); + if (counts_to_1000() != 1000) + return 101; + for (i = 0; i < TRIES; i++) + n += spins(); + return n; +} + +SEC("tc") +int tail_call_after_timeout(struct __sk_buff *skb) +{ + spins(); + bpf_tail_call(skb, &jmp_table, 0); + return 100; +} + +static __noinline int throws(void) +{ + spin(); + bpf_throw(7); + return 0; +} + +SEC("tc") +int throw_after_timeout(void *ctx) +{ + return throws(); +} + +/* + * bpf_user_ringbuf_drain() + */ +struct { + __uint(type, BPF_MAP_TYPE_USER_RINGBUF); + __uint(max_entries, 4096); +} user_ringbuf SEC(".maps"); + +static long drain_cb(struct bpf_dynptr *dynptr, int *spun) +{ + *spun += spin(); + return 0; +} + +SEC("syscall") +int drain(void *ctx) +{ + int spun = 0; + + return bpf_user_ringbuf_drain(&user_ringbuf, drain_cb, &spun, 0) * 10 + spun; +} + +/* + * A tracing prog with large frames: they are in a private stack. + */ +int priv_stack_spun, priv_stack_cb_spun; + +static __noinline int spins_with_stack(void) +{ + volatile char buf[128] = {}; + + return spin() + buf[zero & 127]; +} + +static int priv_stack_cb(__u64 i, int *spun) +{ + volatile char buf[128] = {}; + + *spun += spin() + buf[zero & 127]; + return 0; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(priv_stack) +{ + volatile char buf[128] = {}; + int i, n = buf[zero & 127], spun = 0; + + for (i = 0; i < TRIES; i++) + n += spins_with_stack(); + bpf_loop(TRIES, priv_stack_cb, &spun, zero); + priv_stack_spun = n; + priv_stack_cb_spun = spun; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index e0c389102db9..d908298b9fc7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -903,26 +903,28 @@ __naked void unconditional_jump_to_fill(void) SEC("raw_tp") __arch_x86_64 __log_level(4) -__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 24") -/* may_goto counter at -24 */ -__xlated("0: *(u64 *)(r10 -24) =") -/* may_goto timestamp at -16 */ -__xlated("1: *(u64 *)(r10 -16) =") -__xlated("2: r1 = 1") +__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 8") +/* may_goto counter and timestamp */ +__xlated("0: r0 = ") +__xlated("2: r0 = &(void __percpu *)(r0)") __xlated("...") -__xlated("4: r0 = &(void __percpu *)(r0)") +__xlated("6: r1 = 1") __xlated("...") -/* may_goto expansion starts */ -__xlated("6: r12 = *(u64 *)(r10 -24)") -__xlated("7: if r12 == 0x0 goto pc+6") -__xlated("8: r12 -= 1") -__xlated("9: if r12 != 0x0 goto pc+2") -__xlated("10: r12 = -24") -__xlated("11: call unknown") -__xlated("12: *(u64 *)(r10 -24) = r12") +__xlated("8: r0 = &(void __percpu *)(r0)") +__xlated("...") +/* may_goto expansion starts, r0 and r1 are live */ +__xlated("10: r2 = ") +__xlated("12: r2 = &(void __percpu *)(r2)") +__xlated("13: r12 = *(u64 *)(r2 +0)") +__xlated("14: if r12 == 0x0 goto pc+6") +__xlated("15: r12 -= 1") +__xlated("16: if r12 != 0x0 goto pc+2") +__xlated("17: r12 = r2") +__xlated("18: call unknown") +__xlated("19: *(u64 *)(r2 +0) = r12") /* may_goto expansion ends */ -__xlated("13: *(u64 *)(r10 -8) = r1") -__xlated("14: exit") +__xlated("20: *(u64 *)(r10 -8) = r1") +__xlated("21: exit") __success __naked void may_goto_interaction_x86_64(void) { @@ -946,24 +948,26 @@ __arch_arm64 __arch_riscv64 __arch_loongarch __log_level(4) -__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 24") -/* may_goto counter at -24 */ -__xlated("0: *(u64 *)(r10 -24) =") -/* may_goto timestamp at -16 */ -__xlated("1: *(u64 *)(r10 -16) =") -__xlated("2: r1 = 1") -__xlated("3: call bpf_get_smp_processor_id") -/* may_goto expansion starts */ -__xlated("4: r12 = *(u64 *)(r10 -24)") -__xlated("5: if r12 == 0x0 goto pc+6") -__xlated("6: r12 -= 1") -__xlated("7: if r12 != 0x0 goto pc+2") -__xlated("8: r12 = -24") -__xlated("9: call unknown") -__xlated("10: *(u64 *)(r10 -24) = r12") +__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 8") +/* may_goto counter and timestamp */ +__xlated("0: r0 = ") +__xlated("2: r0 = &(void __percpu *)(r0)") +__xlated("...") +__xlated("6: r1 = 1") +__xlated("7: call bpf_get_smp_processor_id") +/* may_goto expansion starts, r0 and r1 are live */ +__xlated("8: r2 = ") +__xlated("10: r2 = &(void __percpu *)(r2)") +__xlated("11: r12 = *(u64 *)(r2 +0)") +__xlated("12: if r12 == 0x0 goto pc+6") +__xlated("13: r12 -= 1") +__xlated("14: if r12 != 0x0 goto pc+2") +__xlated("15: r12 = r2") +__xlated("16: call unknown") +__xlated("17: *(u64 *)(r2 +0) = r12") /* may_goto expansion ends */ -__xlated("11: *(u64 *)(r10 -8) = r1") -__xlated("12: exit") +__xlated("18: *(u64 *)(r10 -8) = r1") +__xlated("19: exit") __success __naked void may_goto_interaction(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_callx.c b/tools/testing/selftests/bpf/progs/verifier_callx.c index 7e59188d9fca..a12bc421e75c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_callx.c +++ b/tools/testing/selftests/bpf/progs/verifier_callx.c @@ -289,6 +289,26 @@ __naked void callx_callee_writes_caller_stack(void) : __clobber_all); } +/* callee can store a pointer that is not FP-derived over a pointer spilled by the caller */ +SEC("socket") +__success __log_level(2) +__msg("r0 = *(u64 *)(r10 -16) // fp-16=fp0-8|fp0+0") +__naked void callx_callee_can_overwrite_spilled_fp(void) +{ + asm volatile ( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" + "r2 = %[write42] ll;" + "callx r2;" + "r0 = *(u64 *)(r10 - 16);" + "r0 = 0;" + "exit;" + : + : __imm_addr(write42) + : __clobber_all); +} + SEC("socket") __failure __msg("R1 has type scalar, expected func") __naked void callx_scalar(void) diff --git a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c index 574b26b5a7df..5e8461b2167d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_subprogs.c @@ -538,14 +538,15 @@ int return_from_void_global(struct __sk_buff *skb) int global_calls_loop(int x); -static __noinline int static_calls_global(int x) +static __noinline void static_calls_global(int x) { - return global_calls_loop(x); + global_calls_loop(x); } static __noinline int loop_cb_calls_static(u32 i, void *ctx) { - return static_calls_global(i); + static_calls_global(i); + return 0; } __noinline int global_calls_loop(int x) diff --git a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c index 1fbcc5228306..4dd98acc6436 100644 --- a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c +++ b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c @@ -553,7 +553,11 @@ int cond_break5(const void *ctx) { int cnt1 = zero, cnt2; - for (;;) { + /* + * There is one time slice for all may_goto of the prog. When it's used up a function that + * is called after that makes no iteration. So this loop has an end. + */ + for (; cnt1 < 100;) { cond_break; cnt1++; } diff --git a/tools/testing/selftests/bpf/progs/verifier_live_stack.c b/tools/testing/selftests/bpf/progs/verifier_live_stack.c index 5f9b11a81260..2ae5c60150dd 100644 --- a/tools/testing/selftests/bpf/progs/verifier_live_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_live_stack.c @@ -1943,11 +1943,15 @@ static __used __naked void grandchild_deref(void) ::: __clobber_all); } +/* + * The callee gets a pointer to the frame that holds the spills. It can store a pointer + * that is not FP-derived over them: after the call each of them is "fp-N or not FP-derived". + */ SEC("socket") __log_level(2) __success -__msg("14: (79) r1 = *(u64 *)(r10 -8) // r6=fp0-8 r7=fp1-16 fp-8=fp1-16 fp-16=fp0-8") -__msg("15: (79) r0 = *(u64 *)(r1 +0) // r1=fp1-16 r6=fp0-8 r7=fp1-16 fp-8=fp1-16 fp-16=fp0-8") +__msg("14: (79) r1 = *(u64 *)(r10 -8) // r6=fp0-8 r7=fp1-16 fp-8=fp1-16|fp1+0 fp-16=fp0-8|fp0+0") +__msg("15: (79) r0 = *(u64 *)(r1 +0) // r1=fp1-16|fp1+0 r6=fp0-8 r7=fp1-16 fp-8=fp1-16|fp1+0") __msg("stack use/def subprog#1 mid_two_fp_threshold (d1,cs2):") __msg("14: (79) r1 = *(u64 *)(r10 -8) ; use: fp1-8") __msg("15: (79) r0 = *(u64 *)(r1 +0) ; use: fp1-16") @@ -1992,8 +1996,8 @@ static __used __naked void inner_nop_fptest(void) SEC("socket") __log_level(2) __success -__msg("13: (79) r1 = *(u64 *)(r10 -8) // r6=fp0-8 r7=fp1-16 fp-8=fp1-16 fp-16=fp0-8") -__msg("14: (79) r0 = *(u64 *)(r1 +0) // r1=fp1-16 r6=fp0-8 r7=fp1-16 fp-8=fp1-16 fp-16=fp0-8") +__msg("13: (79) r1 = *(u64 *)(r10 -8) // r6=fp0-8 r7=fp1-16 fp-8=fp1-16|fp1+0 fp-16=fp0-8|fp0+0") +__msg("14: (79) r0 = *(u64 *)(r1 +0) // r1=fp1-16|fp1+0 r6=fp0-8 r7=fp1-16 fp-8=fp1-16|fp1+0") __msg("stack use/def subprog#1 mid_one_fp_threshold (d1,cs2):") __msg("13: (79) r1 = *(u64 *)(r10 -8) ; use: fp1-8") __msg("14: (79) r0 = *(u64 *)(r1 +0) ; use: fp1-16") @@ -2565,6 +2569,189 @@ __naked void stack_or_non_stack_write(void) : __clobber_all); } +/* @insns start with fp-8 = 0 and r0 = pointer to map value */ +#define WITH_MAP_VALUE(insns) \ + asm volatile ( \ + "*(u64 *)(r10 - 8) = 0;" \ + "r1 = %[array_map_8b] ll;" \ + "r2 = r10;" \ + "r2 += -8;" \ + "call %[bpf_map_lookup_elem];" \ + "if r0 == 0 goto 9f;" \ + insns \ +"9:" \ + "exit;" \ + : \ + : __imm(bpf_map_lookup_elem), \ + __imm(bpf_loop), \ + __imm_addr(array_map_8b) \ + : __clobber_all) + +/* + * r1 and r2 are r10 or a pointer to map value, joined in both orders. The stores through them + * are not certain writes to fp-8 and fp-16: both slots are live before the stores. + */ +SEC("socket") +__success __log_level(2) +__msg("r1: fp0+0 + _ => fp0 ?") +__msg("r2: _ + fp0+0 => fp0 ?") +__naked void join_of_r10_and_map_value(void) +{ + WITH_MAP_VALUE( + "*(u64 *)(r10 - 16) = 0;" + "r1 = r10;" + "r2 = r0;" + "r2 += 16;" + "r3 = *(u64 *)(r0 + 0);" + "if r3 == 0 goto 1f;" + "r1 = r0;" + "r1 += 8;" + "r2 = r10;" +"1:" + "*(u64 *)(r1 - 8) = 42;" + "*(u64 *)(r2 - 16) = 42;" + "r0 = *(u64 *)(r10 - 8);" + "r1 = *(u64 *)(r10 - 16);"); +} + +/* The counter in fp-8 is the offset in the map value r9. The store through r8 can change it. */ +#define LOOP_WITH_STORE_THROUGH_R8 \ +"8:" \ + "r1 = *(u64 *)(r10 - 8);" \ + "r2 = r9;" \ + "r2 += r1;" \ + "r0 = *(u8 *)(r2 + 0);" \ + "r1 += 1;" \ + "*(u64 *)(r10 - 8) = r1;" \ + "*(u64 *)(r8 + 0) = 4096;" \ + "if r1 < 8 goto 8b;" + +/* + * r7 is r10 or a pointer to map value. The store through r7 is not a certain write to fp-16: + * fp-16 can still hold &fp[-8], and then the store through r8 changes the counter of the loop. + */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=4096 size=1") +__naked void join_of_r10_and_map_value_hidden_store(void) +{ + WITH_MAP_VALUE( + "r9 = r0;" + "r7 = r10;" + "r1 = *(u64 *)(r9 + 0);" + "if r1 == 0 goto 1f;" + "r7 = r9;" + "r7 += 16;" +"1:" + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" + "*(u64 *)(r7 - 16) = r9;" + "r8 = *(u64 *)(r10 - 16);" + LOOP_WITH_STORE_THROUGH_R8); +} + +/* ctx[1] = ctx[0]. ctx is the second param, as in a callback of bpf_loop(). */ +static __used __naked void copy_first_to_second(void) +{ + asm volatile ( + "r1 = *(u64 *)(r2 + 0);" + "*(u64 *)(r2 + 8) = r1;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * fp-16 holds &fp[-8]. The callee gets &fp[-24] and stores a pointer to map value to fp-16. + * The store through the filled pointer is not a certain write to fp-8: fp-8 is live before it. + */ +SEC("socket") +__success __retval(0) __log_level(2) +__msg("r1 = *(u64 *)(r10 -16) // fp-16=fp0-8|fp0+0") +__msg("*(u64 *)(r1 +0) = 42{{ +}}; may_def: fp0-8") +__naked void callee_overwrites_spilled_fp(void) +{ + WITH_MAP_VALUE( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" + "*(u64 *)(r10 - 24) = r0;" + "r2 = r10;" + "r2 += -24;" + "call copy_first_to_second;" + "r1 = *(u64 *)(r10 - 16);" + "*(u64 *)(r1 + 0) = 42;" + "r0 = *(u64 *)(r10 - 8);"); +} + +/* The same, but fp-16 holds a copy of r10, and the callee stores map value + 8 over it. */ +SEC("socket") +__success __retval(0) +__naked void callee_overwrites_spilled_r10(void) +{ + WITH_MAP_VALUE( + "*(u64 *)(r10 - 16) = r10;" + "r0 += 8;" + "*(u64 *)(r10 - 24) = r0;" + "r2 = r10;" + "r2 += -24;" + "call copy_first_to_second;" + "r1 = *(u64 *)(r10 - 16);" + "*(u64 *)(r1 - 8) = 42;" + "r0 = *(u64 *)(r10 - 8);"); +} + +/* The same, but the callee is a callback. */ +SEC("socket") +__success __retval(0) +__load_if_JITed() +__naked void callback_overwrites_spilled_fp(void) +{ + WITH_MAP_VALUE( + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" + "*(u64 *)(r10 - 24) = r0;" + "r1 = 1;" + "r2 = copy_first_to_second ll;" + "r3 = r10;" + "r3 += -24;" + "r4 = 0;" + "call %[bpf_loop];" + "r1 = *(u64 *)(r10 - 16);" + "*(u64 *)(r1 + 0) = 42;" + "r0 = *(u64 *)(r10 - 8);"); +} + +/* + * fp-16 holds &fp[-8], fp-24 holds &fp[-16], and the callee stores a pointer to map value to fp-24. + * fp-16 is still known after the call, and the store through r7 is not a certain write to it: + * r8 is &fp[-8], and the store through r8 changes the counter of the loop. + */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=8 off=4096 size=1") +__naked void callee_overwrites_spilled_fp_hidden_store(void) +{ + WITH_MAP_VALUE( + "r9 = r0;" + "r1 = r10;" + "r1 += -8;" + "*(u64 *)(r10 - 16) = r1;" + "r1 += -8;" + "*(u64 *)(r10 - 24) = r1;" + "*(u64 *)(r10 - 32) = r9;" + "r2 = r10;" + "r2 += -32;" + "call copy_first_to_second;" + /* the fill keeps fp-16 live over the call */ + "r1 = *(u64 *)(r10 - 16);" + "*(u64 *)(r10 - 16) = r1;" + "r7 = *(u64 *)(r10 - 24);" + "*(u64 *)(r7 + 0) = r9;" + "r8 = *(u64 *)(r10 - 16);" + LOOP_WITH_STORE_THROUGH_R8); +} + SEC("socket") __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index faf9eec0a63e..20152a15ca36 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -82,23 +82,31 @@ __naked void may_goto_batch_1(void) SEC("raw_tp") __description("may_goto batch with offsets 2/0") +__load_if_JITed() __arch_x86_64 __arch_s390x __arch_arm64 __arch_riscv64 __arch_loongarch -__xlated("0: *(u64 *)(r10 -16) = 65535") -__xlated("1: *(u64 *)(r10 -8) = 0") -__xlated("2: r12 = *(u64 *)(r10 -16)") -__xlated("3: if r12 == 0x0 goto pc+6") -__xlated("4: r12 -= 1") -__xlated("5: if r12 != 0x0 goto pc+2") -__xlated("6: r12 = -16") -__xlated("7: call unknown") -__xlated("8: *(u64 *)(r10 -16) = r12") -__xlated("9: r0 = 1") -__xlated("10: r0 = 2") -__xlated("11: exit") +/* the main prog sets count, timestamp, owner */ +__xlated("0: r0 = ") +__xlated("2: r0 = &(void __percpu *)(r0)") +__xlated("3: *(u64 *)(r0 +0) = 65535") +__xlated("4: *(u64 *)(r0 +8) = 0") +__xlated("5: *(u64 *)(r0 +16) = 0") +/* may_goto, r0 is dead */ +__xlated("6: r0 = ") +__xlated("8: r0 = &(void __percpu *)(r0)") +__xlated("9: r12 = *(u64 *)(r0 +0)") +__xlated("10: if r12 == 0x0 goto pc+6") +__xlated("11: r12 -= 1") +__xlated("12: if r12 != 0x0 goto pc+2") +__xlated("13: r12 = r0") +__xlated("14: call unknown") +__xlated("15: *(u64 *)(r0 +0) = r12") +__xlated("16: r0 = 1") +__xlated("17: r0 = 2") +__xlated("18: exit") __success __naked void may_goto_batch_2(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_private_stack.c b/tools/testing/selftests/bpf/progs/verifier_private_stack.c index 3d9994a52464..bbc09f516d92 100644 --- a/tools/testing/selftests/bpf/progs/verifier_private_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_private_stack.c @@ -279,6 +279,105 @@ int private_stack_exception_sub_prog(void) return 0; } +/* + * Nothing attaches the two fentry progs above, so they are loaded, but never + * run. BPF_PROG_TEST_RUN does run a raw_tp prog. The default exception callback + * returns the cookie of bpf_throw(). + */ +SEC("raw_tp") +__description("Private stack, run, exception in main prog") +__success __retval(1) +__arch_x86_64 +__jited(" movq %rdi, -0x200(%r9)") +__jited(" movl $0x1, %edi") +__jited(" pushq %r9") +__jited("...") +__jited(" callq") +__naked void private_stack_run_exception_main_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 1; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +__used +__naked static void subprog_throw(void) +{ + asm volatile (" \ + r1 = 2; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +SEC("raw_tp") +__description("Private stack, run, exception in subprog") +__success __retval(2) +__arch_x86_64 +__jited(" movq %rdi, -0x200(%r9)") +__jited(" pushq %r9") +__jited("...") +__jited(" callq") +__jited(" popq %r9") +__naked void private_stack_run_exception_sub_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + call subprog_throw; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +__used +__naked static void private_stack_subprog_throw(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 512) = r1; \ + r1 = 3; \ + call %[bpf_throw]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_throw) + : __clobber_all); +} + +/* Only the frame of the main prog counts, and it has nothing pushed at the call */ +SEC("raw_tp") +__description("Private stack, run, exception in subprog, no private stack in main prog") +__success __retval(3) +__arch_x86_64 +__jited(" subq $0x8, %rsp") +__jited("...") +__jited("func #1") +__jited("...") +__jited(" movq %rdi, -0x200(%r9)") +__jited(" movl $0x3, %edi") +__jited(" pushq %r9") +__naked void no_private_stack_run_exception_sub_prog(void) +{ + asm volatile (" \ + r1 = 42; \ + *(u64 *)(r10 - 8) = r1; \ + call private_stack_subprog_throw; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + int glob; __noinline static void subprog2(int *val) { diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index fa01a616af9d..0be4fe8aadc7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -3936,4 +3936,186 @@ __naked void clamp_pkt_pointer_keeps_range_of_entry(void) : __clobber_all); } +/* + * r4 is 0, 8 .. 48, but its umax is 50. @bytes from r6 are in the packet: the range of r6 is + * @start + 50 + @bytes. + */ +#define PKT_PLUS_R4(start, bytes) \ + "r7 = *(u32 *)(r1 + %[data_end]);" \ + "r6 = *(u32 *)(r1 + %[data]);" \ + "r4 = *(u32 *)(r1 + %[mark]);" \ + "r4 &= 0x38;" \ + "if r4 > 50 goto 9f;" \ + "r6 += " #start ";" \ + "r6 += r4;" \ + "r5 = r6;" \ + "r5 += " #bytes ";" \ + "if r5 > r7 goto 9f;" + +#define PKT_IMMS \ + __imm_const(data, offsetof(struct __sk_buff, data)), \ + __imm_const(data_end, offsetof(struct __sk_buff, data_end)), \ + __imm_const(mark, offsetof(struct __sk_buff, mark)) + +/* + * r6 is 0..58 at the head, and 0..56 with the step of 8. The range must go from 58 to 56 too: + * in the second iteration no byte from r6 is in the packet. + */ +SEC("tc") +__failure __msg("invalid access to packet, off=56 size=2, R6(id=2,off=56,r=56)") +__naked void widen_pkt_pointer_step_lowers_umax(void) +{ + asm volatile ( + PKT_PLUS_R4(0, 8) + "r9 = 0;" +"1:" + "r0 = *(u16 *)(r6 + 0);" + "r6 += 8;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + :: PKT_IMMS : __clobber_all); +} + +/* The pointer goes down. 8 bytes from the first r6 are in the packet, not 10. */ +SEC("tc") +__failure __msg("invalid access to packet, off=120 size=2, R6(id=2,off=120,r=120)") +__naked void widen_pkt_pointer_step_lowers_umax_down(void) +{ + asm volatile ( + PKT_PLUS_R4(64, 8) + "r9 = 0;" +"1:" + "r0 = *(u16 *)(r6 + 8);" + "r6 += -8;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + :: PKT_IMMS : __clobber_all); +} + +/* + * r6 is -8..50 at the head, and -8..48 with the step of 8. Its umax is U64_MAX in the two cases. + * r5 is 0..56 and has the range of r6. In the first iteration no byte from r5 is in the packet. + */ +SEC("tc") +__failure __msg("invalid access to packet, off=56 size=2, R5(id=2,off=56,r=56)") +__naked void widen_pkt_pointer_negative_offset(void) +{ + asm volatile ( + PKT_PLUS_R4(0, 8) + "r9 = 0;" +"1:" + "r5 = r6;" + "r5 += 8;" + "r0 = *(u16 *)(r5 + 0);" + "r6 += -8;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + :: PKT_IMMS : __clobber_all); +} + +/* The access is in the second iteration only: the verifier gets to it after the clamp */ +SEC("tc") +__failure __msg("invalid access to packet, off=56 size=2, R6(id=3,off=56,r=56)") +__naked void clamp_pkt_pointer_step_lowers_umax(void) +{ + asm volatile ( + PKT_PLUS_R4(0, 8) + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w1 = *(u32 *)(r10 - 8);" + "if w1 == 0 goto 2f;" + "r0 = *(u16 *)(r6 + 0);" +"2:" + "w1 = 1;" + "*(u32 *)(r10 - 8) = r1;" + "r6 += 8;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + :: PKT_IMMS : __clobber_all); +} + +/* r5 is r5 = r6 + 8 or r6 at the head. Here var_off lowers umax. No byte from r6 + 8 is valid. */ +SEC("tc") +__failure __msg("invalid access to packet, off=56 size=2, R5(id=2,off=56,r=56)") +__naked void union_pkt_pointer_var_off_lowers_umax(void) +{ + asm volatile ( + PKT_PLUS_R4(0, 8) + "r9 = 0;" +"1:" + "r0 = *(u16 *)(r5 + 0);" + "r5 = r6;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + :: PKT_IMMS : __clobber_all); +} + +/* 16 bytes from r6 are in the packet: two iterations read 8 bytes each */ +SEC("tc") +__success __retval(0) +__naked void widen_pkt_pointer_keeps_valid_bytes(void) +{ + asm volatile ( + PKT_PLUS_R4(0, 16) + "r9 = 0;" +"1:" + "r0 = *(u64 *)(r6 + 0);" + "r6 += 8;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + :: PKT_IMMS : __clobber_all); +} + +/* + * r6 is beyond pkt_end in front of the loop and comes back into the packet. The mark is not for + * all iterations: the verifier must look at the two branches of the compare in the first pass. + */ +SEC("tc") +__success +__log_level(2) +__msg("widening r6 to 32..96 step 32") +__msg("6: R6=pkt(id=1,r=0,smin=umin=smin32=umin32=32,smax=umax=smax32=umax32=96") +__naked void widen_pkt_pointer_beyond_pkt_end(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[data_end]);" + "r6 = *(u32 *)(r1 + %[data]);" + "r6 += 96;" + "if r6 > r7 goto 1f;" + "goto 9f;" +"1:" + "r9 = 0;" +"2:" + "if r6 > r7 goto 3f;" + "r0 = *(u8 *)(r6 - 1);" +"3:" + "r6 += -32;" + "r9 += 1;" + "if r9 < 3 goto 2b;" +"9:" + "r0 = 0;" + "exit;" + :: PKT_IMMS : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index 15a12497e074..abf319cb8514 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -552,6 +552,121 @@ __naked void stack_arg_may_goto_in_caller(void) ); } +/* arg6 is an offset into the value of map_hash_8b. 0-7 are in bounds. */ +__naked __noinline __used +static long arg6_is_map_value_off(long a, long b, long c, long d, long e, long off) +{ + asm volatile ( + "r6 = *(u64 *)(r11 + 8);" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r2 = r10;" + "r2 += -8;" + "r1 = %[map_hash_8b] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l0_%=;" + "r0 += r6;" + "r0 = *(u8 *)(r0 + 0);" + "l0_%=:" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: a pruned state propagates precision of arg6") +__failure __msg("invalid access to map value") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void stack_arg_precision_pruned_state(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto l0_%=;" + /* Path 1: arg6 = 0. It makes arg6 precise at l3. */ + "*(u64 *)(r11 - 8) = 0;" + "goto l3_%=;" + "l0_%=:" + "if r0 == 1 goto l1_%=;" + /* Path 2: arg6 = r6 = 0. Pruned at l3. It has to make r6 precise at l2. */ + "r6 = 0;" + "goto l2_%=;" + "l1_%=:" + /* Path 3: arg6 = r6 = 8. Out of bounds. */ + "r6 = 8;" + "l2_%=:" + "*(u64 *)(r11 - 8) = r6;" + "l3_%=:" + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call arg6_is_map_value_off;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + +/* The same, but backtracking of r1 sees a store through r1 and marks all scalars precise. */ +__naked __noinline __used +static long arg6_is_map_value_off_fallback(long a, long b, long c, long d, long e, long off) +{ + asm volatile ( + "r6 = *(u64 *)(r11 + 8);" + "r1 = 0;" + "*(u64 *)(r10 - 8) = r1;" + "r2 = r10;" + "r2 += -8;" + "r1 = %[map_hash_8b] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l0_%=;" + "r1 = r0;" + "*(u8 *)(r1 + 0) = 0;" + "r1 -= r0;" + "r1 &= 0;" + "r1 += r6;" + "r0 += r1;" + "r0 = *(u8 *)(r0 + 0);" + "l0_%=:" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: forcing all scalars precise covers arg6") +__failure __msg("invalid access to map value") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void stack_arg_precision_fallback(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "if r0 != 0 goto l0_%=;" + "*(u64 *)(r11 - 8) = 0;" + "goto l1_%=;" + "l0_%=:" + /* Out of bounds. Must not be pruned at l1. */ + "*(u64 *)(r11 - 8) = 8;" + "l1_%=:" + "r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "call arg6_is_map_value_off_fallback;" + "exit;" + :: __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + #else SEC("socket")