diff --git a/Documentation/bpf/bpf_iterators.rst b/Documentation/bpf/bpf_iterators.rst index 189e3ec1c6c8..72435e32ffa1 100644 --- a/Documentation/bpf/bpf_iterators.rst +++ b/Documentation/bpf/bpf_iterators.rst @@ -86,6 +86,16 @@ Constructor/next/destructor implementation contract is as follows: resources and marks stack space used by `struct bpf_iter_` as usable for something else. Destructor is marked with KF_ITER_DESTROY flag. +The verifier treats a non-RCU, non-acquired struct result as borrowed from +the iterator's current element. It invalidates the previous result and its +tracked derived pointers on the next call, including when it returns NULL, +and on destruction of the iterator. Resources borrowing that lifetime must +be released first. A next method whose struct result has an independent +lifetime must use ``KF_ITER_RET_INDEPENDENT``. This flag requires +``KF_ITER_NEXT`` and a struct pointer return type, and cannot be combined +with ``KF_ACQUIRE``. RCU-protected and acquired results retain their own +lifetime rules. + Any open-coded BPF iterator implementation has to implement at least these three methods. It is enforced that for any given type of iterator only applicable constructor/destructor/next are callable. I.e., verifier ensures diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index aaa34c739004..33632fb20997 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -3502,11 +3502,6 @@ bool bpf_jit_supports_arena(void) return true; } -bool bpf_jit_supports_arena_scalar(void) -{ - return true; -} - bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { if (!in_arena) diff --git a/arch/arm64/net/bpf_timed_may_goto.S b/arch/arm64/net/bpf_timed_may_goto.S index a9a802711a7f..1b4a4f91ffc9 100644 --- a/arch/arm64/net/bpf_timed_may_goto.S +++ b/arch/arm64/net/bpf_timed_may_goto.S @@ -5,13 +5,16 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* Allocate stack space and emit frame record */ - stp x29, x30, [sp, #-64]! + stp x29, x30, [sp, #-96]! mov x29, sp /* Save BPF registers R0 - R5 (x8, x0-x4)*/ stp x8, x0, [sp, #16] stp x1, x2, [sp, #32] stp x3, x4, [sp, #48] + /* Save stack arguments 6 - 8 (x5-x7) */ + stp x5, x6, [sp, #64] + str x7, [sp, #80] /* * Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP @@ -32,9 +35,12 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) ldp x8, x0, [sp, #16] ldp x1, x2, [sp, #32] ldp x3, x4, [sp, #48] + /* Restore stack arguments 6 - 8 (x5-x7) */ + ldp x5, x6, [sp, #64] + ldr x7, [sp, #80] /* Restore FP and LR */ - ldp x29, x30, [sp], #64 + ldp x29, x30, [sp], #96 ret SYM_FUNC_END(arch_bpf_timed_may_goto) diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index fcf850f244c9..6ec37f634aff 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -10,15 +10,16 @@ * - output: updated count in BPF_REG_AX (t0) * * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) are saved - * across the call; BPF_REG_FP (s5) is callee-saved and needs no saving. + * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) and stack + * arguments 6-8 (a5-a7) are saved across the call; BPF_REG_FP (s5) is + * callee-saved and needs no saving. */ SYM_FUNC_START(arch_bpf_timed_may_goto) - addi sp, sp, -(8*SZREG) - REG_S ra, 7*SZREG(sp) - REG_S s0, 6*SZREG(sp) - addi s0, sp, 8*SZREG + addi sp, sp, -(12*SZREG) + REG_S ra, 11*SZREG(sp) + REG_S s0, 10*SZREG(sp) + addi s0, sp, 12*SZREG /* Save BPF registers R0-R5 (t6, a0-a4) */ REG_S t6, 5*SZREG(sp) @@ -27,6 +28,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S a2, 2*SZREG(sp) REG_S a3, 1*SZREG(sp) REG_S a4, 0*SZREG(sp) + /* Save stack arguments 6-8 (a5-a7) */ + REG_S a5, 6*SZREG(sp) + REG_S a6, 7*SZREG(sp) + REG_S a7, 8*SZREG(sp) add a0, t0, s5 call bpf_check_timed_may_goto @@ -39,9 +44,13 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_L a1, 3*SZREG(sp) REG_L a0, 4*SZREG(sp) REG_L t6, 5*SZREG(sp) + /* Restore stack arguments 6-8 */ + REG_L a5, 6*SZREG(sp) + REG_L a6, 7*SZREG(sp) + REG_L a7, 8*SZREG(sp) - REG_L s0, 6*SZREG(sp) - REG_L ra, 7*SZREG(sp) - addi sp, sp, 8*SZREG + REG_L s0, 10*SZREG(sp) + REG_L ra, 11*SZREG(sp) + addi sp, sp, 12*SZREG ret SYM_FUNC_END(arch_bpf_timed_may_goto) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 6a2fe3d0039d..d90b6d238b3a 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -97,56 +97,6 @@ static bool is_imm8(int value) return value <= 127 && value >= -128; } -/* - * Let us limit the positive offset to be <= 123. - * This is to ensure eventual jit convergence For the following patterns: - * ... - * pass4, final_proglen=4391: - * ... - * 20e: 48 85 ff test rdi,rdi - * 211: 74 7d je 0x290 - * 213: 48 8b 77 00 mov rsi,QWORD PTR [rdi+0x0] - * ... - * 289: 48 85 ff test rdi,rdi - * 28c: 74 17 je 0x2a5 - * 28e: e9 7f ff ff ff jmp 0x212 - * 293: bf 03 00 00 00 mov edi,0x3 - * Note that insn at 0x211 is 2-byte cond jump insn for offset 0x7d (-125) - * and insn at 0x28e is 5-byte jmp insn with offset -129. - * - * pass5, final_proglen=4392: - * ... - * 20e: 48 85 ff test rdi,rdi - * 211: 0f 84 80 00 00 00 je 0x297 - * 217: 48 8b 77 00 mov rsi,QWORD PTR [rdi+0x0] - * ... - * 28d: 48 85 ff test rdi,rdi - * 290: 74 1a je 0x2ac - * 292: eb 84 jmp 0x218 - * 294: bf 03 00 00 00 mov edi,0x3 - * Note that insn at 0x211 is 6-byte cond jump insn now since its offset - * becomes 0x80 based on previous round (0x293 - 0x213 = 0x80). - * At the same time, insn at 0x292 is a 2-byte insn since its offset is - * -124. - * - * pass6 will repeat the same code as in pass4 and this will prevent - * eventual convergence. - * - * To fix this issue, we need to break je (2->6 bytes) <-> jmp (5->2 bytes) - * cycle in the above. In the above example je offset <= 0x7c should work. - * - * For other cases, je <-> je needs offset <= 0x7b to avoid no convergence - * issue. For jmp <-> je and jmp <-> jmp cases, jmp offset <= 0x7c should - * avoid no convergence issue. - * - * Overall, let us limit the positive offset for 8bit cond/uncond jmp insn - * to maximum 123 (0x7b). This way, the jit pass can eventually converge. - */ -static bool is_imm8_jmp_offset(int value) -{ - return value <= 123 && value >= -128; -} - static bool is_simm32(s64 value) { return value == (s64)(s32)value; @@ -1795,26 +1745,20 @@ static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size, /* * A fetching AND/OR/XOR can't be implemented with a single x86 insn, so do a * CMPXCHG loop. @index_reg is X86_REG_R12 for an arena access or -1 otherwise. - * Unless @r0_on_stack, the caller has substituted BPF_REG_AX for R0 in - * @dst_reg and @src_reg. + * The caller has saved R0 in BPF_REG_AX and has substituted BPF_REG_AX for R0 + * in @dst_reg and @src_reg. * * Both the load and the CMPXCHG can fault when accessing the arena, so their * addresses are handed back in @fault[0] and @fault[1]. A fault has to resume * at @resume, which is past the loop and past the move that delivers the old * value, but before R0 is restored. Both are NULL for the non-arena case, * which needs no fixup. - * - * R0 is expected to be saved in BPF_REG_AX by the caller, or pushed on the - * stack if @r0_on_stack. In that case an R0 operand stays in the stack slot, - * RAX ends up holding the old value that R0 is to return, and the slot is - * dropped rather than popped. */ static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, - u32 src_reg, int index_reg, bool r0_on_stack, + u32 src_reg, int index_reg, u8 **fault, u8 **resume) { bool is64 = BPF_SIZE(insn->code) == BPF_DW; - bool operand_on_stack = r0_on_stack && src_reg == BPF_REG_0; u8 *branch_target, *prog = *pprog; int err; @@ -1833,15 +1777,7 @@ static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, * Perform the (commutative) operation locally, put the result in * the AUX_REG. */ - if (operand_on_stack) { - /* mov r11, [rsp]: the operand is the R0 saved there, RAX is the old value */ - if (is64) - EMIT4(0x4C, 0x8B, 0x1C, 0x24); - else - EMIT4(0x44, 0x8B, 0x1C, 0x24); - } else { - emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); - } + emit_mov_reg(&prog, is64, AUX_REG, BPF_REG_0); maybe_emit_mod(&prog, AUX_REG, src_reg, is64); EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)], add_2reg(0xC0, AUX_REG, src_reg)); @@ -1860,20 +1796,14 @@ static int emit_atomic_fetch_rmw(u8 **pprog, struct bpf_insn *insn, u32 dst_reg, /* ZF tells us whether we won the race. If it's cleared we need to try again. */ EMIT2(X86_JNE, -(prog - branch_target) - 2); - /* Return the pre-modification value, which RAX already is for R0 */ - if (!operand_on_stack) - emit_mov_reg(&prog, is64, src_reg, BPF_REG_0); + /* Return the pre-modification value */ + emit_mov_reg(&prog, is64, src_reg, BPF_REG_0); if (resume) *resume = prog; /* Restore R0 after clobbering RAX */ - if (!r0_on_stack) - emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); - else if (operand_on_stack) - EMIT2(0x41, 0x5B); /* pop r11, dropping the operand */ - else - EMIT1(0x58); /* pop rax */ + emit_mov_reg(&prog, true, BPF_REG_0, BPF_REG_AX); *pprog = prog; return 0; @@ -2298,6 +2228,17 @@ static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr) #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp))) +/* + * addrs[] below i are from this pass, addrs[i] and above are from the previous + * one. 'moved' is how far back this pass moved the start of the current insn. + * With it a jump in either direction is sized by the insns it jumps over, plus + * its own previous size for a backward jump, none of which grow. + */ +static int jmp_distance(const int *addrs, int i, int off, int moved) +{ + return addrs[i + off] - addrs[i] + (off < 0 ? moved : 0); +} + #define __LOAD_TCC_PTR(off) \ emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, off) /* mov rax, qword ptr [rbp - rounded_stack_depth - 16] */ @@ -2564,9 +2505,7 @@ static int emit_unwind_ret_addr(u8 **pprog, u8 *ip) EMIT2(0x50, 0x56); /* push rax; push rsi */ EMIT4(0x48, 0x8B, 0x7D, 0x08); /* mov rdi, [rbp + 8] */ - ip += prog - start; - ip += x86_call_depth_emit_accounting(&prog, bpf_unwind_target, ip); - if (emit_call(&prog, bpf_unwind_target, ip)) + if (emit_rsb_call(&prog, bpf_unwind_target, ip + (prog - start))) return -EINVAL; EMIT4(0x48, 0x89, 0x45, 0x08); /* mov [rbp + 8], rax */ EMIT2(0x5E, 0x58); /* pop rsi; pop rax */ @@ -2588,7 +2527,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * u64 arena_vm_start = 0, user_vm_start = 0; void __percpu *priv_stack_ptr; int i, excnt = 0; - int ilen, proglen = 0; + int ilen, moved, proglen = 0; u8 *ip, *prog = temp; u32 stack_depth; int callee_saved_size; @@ -2702,6 +2641,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * if (rw_image) memcpy(rw_image + proglen, temp, ilen); proglen += ilen; + moved = addrs[0] - proglen; addrs[0] = proglen; prog = temp; @@ -3394,7 +3334,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * } if (is_atomic_fetch) { err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, - real_src_reg, -1, false, + real_src_reg, -1, NULL, NULL); if (err) return err; @@ -3423,29 +3363,19 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: if (is_atomic_fetch_op(insn)) { u32 real_src_reg = src_reg, real_dst_reg = dst_reg; - /* - * An access through a number has its address in - * BPF_REG_AX (see arena_scalar_access()), so R0 is - * saved on the stack instead, as div and mod do. - */ - bool r0_on_stack = dst_reg == BPF_REG_AX; u8 *fault[2], *resume; int j; /* Will need RAX as a CMPXCHG operand so save R0 */ - if (r0_on_stack) { - EMIT1(0x50); /* push rax */ - } else { - emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); - if (src_reg == BPF_REG_0) - real_src_reg = BPF_REG_AX; - if (dst_reg == BPF_REG_0) - real_dst_reg = BPF_REG_AX; - } + emit_mov_reg(&prog, true, BPF_REG_AX, BPF_REG_0); + if (src_reg == BPF_REG_0) + real_src_reg = BPF_REG_AX; + if (dst_reg == BPF_REG_0) + real_dst_reg = BPF_REG_AX; err = emit_atomic_fetch_rmw(&prog, insn, real_dst_reg, real_src_reg, X86_REG_R12, - r0_on_stack, fault, &resume); + fault, &resume); if (err) return err; @@ -3747,8 +3677,8 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * default: /* to silence GCC warning */ return -EFAULT; } - jmp_offset = addrs[i + insn->off] - addrs[i]; - if (is_imm8_jmp_offset(jmp_offset)) { + jmp_offset = jmp_distance(addrs, i, insn->off, moved); + if (is_imm8(jmp_offset)) { if (jmp_padding) { /* To keep the jmp_offset valid, the extra bytes are * padded before the jump insn, so we subtract the @@ -3788,23 +3718,10 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * break; case BPF_JMP | BPF_JA: case BPF_JMP32 | BPF_JA: - if (BPF_CLASS(insn->code) == BPF_JMP) { - if (insn->off == -1) - /* -1 jmp instructions will always jump - * backwards two bytes. Explicitly handling - * this case avoids wasting too many passes - * when there are long sequences of replaced - * dead code. - */ - jmp_offset = -2; - else - jmp_offset = addrs[i + insn->off] - addrs[i]; - } else { - if (insn->imm == -1) - jmp_offset = -2; - else - jmp_offset = addrs[i + insn->imm] - addrs[i]; - } + if (BPF_CLASS(insn->code) == BPF_JMP) + jmp_offset = jmp_distance(addrs, i, insn->off, moved); + else + jmp_offset = jmp_distance(addrs, i, insn->imm, moved); if (!jmp_offset) { /* @@ -3833,7 +3750,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * break; } emit_jmp: - if (is_imm8_jmp_offset(jmp_offset)) { + if (is_imm8(jmp_offset)) { if (jmp_padding) { /* To avoid breaking jmp_offset, the extra bytes * are padded before the actual jmp insn, so @@ -3869,7 +3786,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * emit_unwind_ret_addr(&prog, ip)) return -EINVAL; if (seen_exit) { - jmp_offset = ctx->cleanup_addr - addrs[i]; + jmp_offset = ctx->cleanup_addr - addrs[i] + moved; goto emit_jmp; } seen_exit = true; @@ -3930,6 +3847,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * memcpy(rw_image + proglen, temp, ilen); } proglen += ilen; + moved = addrs[i] - proglen; addrs[i] = proglen; prog = temp; } @@ -4967,20 +4885,23 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr /* * Before first pass, make a rough estimation of addrs[] - * each BPF instruction is translated to less than 64 bytes + * each BPF instruction is translated to at most BPF_MAX_INSN_SIZE + * bytes, so no insn grows in the first pass */ for (proglen = 0, i = 0; i <= prog->len; i++) { - proglen += 64; + proglen += BPF_MAX_INSN_SIZE; addrs[i] = proglen; } ctx.cleanup_addr = proglen; skip_init_addrs: /* - * JITed image shrinks with every pass and the loop iterates - * until the image stops shrinking. Very large BPF programs - * may converge on the last pass. In such case do one more - * pass to emit the final image. + * Jumps are sized by the insns they jump over, which start at + * their maximum size, so no insn grows from one pass to the + * next. The loop iterates until the image stops shrinking, at + * which point no insn has moved. Very large BPF programs may + * converge on the last pass. In such case do one more pass to + * emit the final image. */ for (pass = 0; pass < MAX_PASSES || image; pass++) { if (!padding && pass >= PADDING_PASSES) @@ -5289,11 +5210,6 @@ bool bpf_jit_supports_arena(void) return true; } -bool bpf_jit_supports_arena_scalar(void) -{ - return true; -} - bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return true; diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index 6c6e03f725a3..dd2dde89836b 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -22,6 +22,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) pushq %rdx pushq %rcx pushq %r8 + /* Save stack argument 6. */ + pushq %r9 /* * r10 has the pointer to count and timestamp. JIT added the frame @@ -36,6 +38,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* BPF_REG_AX=r10 will be stored into count, so move return value to it. */ movq %rax, %r10 + popq %r9 /* Restore r5-r0. */ popq %r8 popq %rcx diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 580f89c450b5..ff0436781293 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1979,7 +1979,7 @@ struct bpf_prog_aux { struct rcu_work rwork; }; struct bpf_stream *stream[2]; - struct mutex st_ops_assoc_mutex; + spinlock_t st_ops_assoc_lock; /* protects st_ops_assoc */ struct bpf_map __rcu *st_ops_assoc; }; @@ -2240,9 +2240,11 @@ struct btf_member; * and its progs. It is unnecessary if the @unreg * has waited for the correct rcu gp or the @unreg * has ensured all struct_ops prog has finished running. - * @free_after_mult_rcu_gp: Same as @free_after_tasks_rcu_gp but waiting for - * both tasks_trace_rcu and regular rcu grace period. - * It is usually needed if the struct_ops has sleepable prog. + * @free_after_mult_rcu_gp: Set to true if the cgroup prog array of this + * struct_ops needs a tasks_trace_rcu gp, besides a + * regular rcu gp, before it is freed. It is usually + * needed if the struct_ops has sleepable prog. A + * struct_ops map always waits for both. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5926f13ff43c..7baeb831281f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -782,7 +782,6 @@ struct bpf_insn_aux_data { u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */ u64 is_iter_next:1; /* bpf_iter__next() kfunc call */ u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */ - u64 arena_scalar:1; /* ldx/stx/st/atomic through a number, it's an address in arena */ u64 bb_end:1; /* last instruction of a basic block */ u64 need_scev:1; u64 alu_state:5; /* used in combination with alu_limit */ @@ -1074,7 +1073,6 @@ struct bpf_verifier_env { bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ bool test_reg_invariants; /* fail verification on register invariants violations */ - bool arena_scalar; /* load and store through a number is arena access */ struct bpf_verifier_state *cur_state; /* current verifier state */ /* Search pruning optimization, array of list_heads for * lists of struct bpf_verifier_state_list. @@ -1280,6 +1278,7 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...); void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num); +void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num); #define verifier_bug_if(cond, env, fmt, args...) \ ({ \ @@ -1314,6 +1313,12 @@ static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int ins return env->insn_aux_data[insn_idx].force_checkpoint; } +static inline void clear_scalar_id(struct bpf_reg_state *reg) +{ + reg->id = 0; + reg->delta = 0; +} + static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx) { env->insn_aux_data[idx].calls_callback = true; diff --git a/include/linux/btf.h b/include/linux/btf.h index b66e20a18c54..a18254b7521b 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -82,6 +82,7 @@ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ #define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ #define KF_PERFMON (1 << 18) /* kfunc requires CAP_PERFMON */ +#define KF_ITER_RET_INDEPENDENT (1 << 19) /* next result outlives the current element */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the diff --git a/include/linux/filter.h b/include/linux/filter.h index e0218e677a1d..98254742ae37 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1287,7 +1287,6 @@ bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); -bool bpf_jit_supports_arena_scalar(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); bool bpf_jit_supports_large_stack(void); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index f29b0aad440c..8d7228834305 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -1357,12 +1357,6 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) -/* - * Load and store through a number is an access to the arena of the program - * at the low 32 bits of the number. It's for programs written in Rust. - */ -#define BPF_F_ARENA_SCALAR (1U << 8) - /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ diff --git a/kernel/bpf/bpf_struct_ops.c b/kernel/bpf/bpf_struct_ops.c index 1178acd72296..7c5cd6086e27 100644 --- a/kernel/bpf/bpf_struct_ops.c +++ b/kernel/bpf/bpf_struct_ops.c @@ -1030,6 +1030,7 @@ static void bpf_struct_ops_map_free_pre_rcu(struct bpf_map *map) struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map; bpf_struct_ops_map_del_ksyms(st_map); + bpf_struct_ops_map_dissoc_progs(st_map); } static void bpf_struct_ops_map_free(struct bpf_map *map) @@ -1045,8 +1046,6 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) if (btf_is_module(st_map->btf)) module_put(st_map->st_ops_desc->st_ops->owner); - bpf_struct_ops_map_dissoc_progs(st_map); - if (tasks_rcu && IS_ENABLED(CONFIG_TASKS_RCU)) synchronize_rcu_tasks(); @@ -1162,8 +1161,11 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) mutex_init(&st_map->lock); bpf_map_init_from_attr(map, attr); - map->free_after_mult_rcu_gp = st_ops_desc->st_ops->free_after_mult_rcu_gp; - map->free_after_rcu_gp = true; + /* + * A prog dissociated in map_free_pre_rcu() may still run the struct_ops + * from a bpf_wq callback or a sleepable prog, under RCU tasks trace. + */ + map->free_after_mult_rcu_gp = true; return map; @@ -1541,10 +1543,10 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map) { struct bpf_map *st_ops_assoc; - guard(mutex)(&prog->aux->st_ops_assoc_mutex); + guard(spinlock_irqsave)(&prog->aux->st_ops_assoc_lock); st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc, - lockdep_is_held(&prog->aux->st_ops_assoc_mutex)); + lockdep_is_held(&prog->aux->st_ops_assoc_lock)); if (st_ops_assoc && st_ops_assoc == map) return 0; @@ -1569,19 +1571,20 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map) void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog) { + struct bpf_prog_aux *aux = prog->aux; struct bpf_map *st_ops_assoc; - guard(mutex)(&prog->aux->st_ops_assoc_mutex); + scoped_guard(spinlock_irqsave, &aux->st_ops_assoc_lock) { + st_ops_assoc = rcu_dereference_protected(aux->st_ops_assoc, + lockdep_is_held(&aux->st_ops_assoc_lock)); + if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON) + return; - st_ops_assoc = rcu_dereference_protected(prog->aux->st_ops_assoc, - lockdep_is_held(&prog->aux->st_ops_assoc_mutex)); - if (!st_ops_assoc || st_ops_assoc == BPF_PTR_POISON) - return; + RCU_INIT_POINTER(aux->st_ops_assoc, NULL); + } if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) bpf_map_put(st_ops_assoc); - - RCU_INIT_POINTER(prog->aux->st_ops_assoc, NULL); } /* diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 538129ed2418..1be30487a2fa 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -1752,15 +1752,16 @@ static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t) /* Expand 'types' array */ struct btf_type **new_types; - u32 expand_by, new_size; + u32 expand_by, new_size, max_size; - if (btf->start_id + btf->types_size == BTF_MAX_TYPE) { + if (btf->start_id + btf->types_size >= BTF_MAX_TYPE) { btf_verifier_log(env, "Exceeded max num of types"); return -E2BIG; } + max_size = BTF_MAX_TYPE - btf->start_id; expand_by = max_t(u32, btf->types_size >> 2, 16); - new_size = min_t(u32, BTF_MAX_TYPE, + new_size = min_t(u32, max_size, btf->types_size + expand_by); new_types = kvzalloc_objs(*new_types, new_size, @@ -5871,6 +5872,7 @@ static int btf_check_all_metas(struct btf_verifier_env *env) struct btf *btf = env->btf; struct btf_header *hdr; void *cur, *end; + int err; hdr = &btf->hdr; cur = btf->nohdr_data + hdr->type_off; @@ -5885,7 +5887,10 @@ static int btf_check_all_metas(struct btf_verifier_env *env) if (meta_size < 0) return meta_size; - btf_add_type(env, t); + err = btf_add_type(env, t); + if (err) + return err; + cur += meta_size; env->log_type_id++; } @@ -9716,6 +9721,10 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, if (!flags || (flags & (flags - 1))) return -EINVAL; + if ((func_flags & KF_ITER_RET_INDEPENDENT) && + (!(flags & KF_ITER_NEXT) || (func_flags & KF_ACQUIRE))) + return -EINVAL; + /* any BPF iter kfunc should have `struct bpf_iter_ *` first arg */ nr_args = btf_type_vlen(func); if (nr_args < 1) @@ -9756,6 +9765,12 @@ static int btf_check_iter_kfuncs(struct btf *btf, const char *func_name, t = btf_type_skip_modifiers(btf, func->type, NULL); if (!t || !btf_type_is_ptr(t)) return -EINVAL; + /* The opt-out applies only to non-acquired struct results. */ + if (func_flags & KF_ITER_RET_INDEPENDENT) { + t = btf_type_skip_modifiers(btf, t->type, NULL); + if (!t || !__btf_type_is_struct(t)) + return -EINVAL; + } } if (flags & KF_ITER_DESTROY) { @@ -9822,7 +9837,8 @@ static int btf_check_kfunc_protos(struct btf *btf, u32 func_id, u32 func_flags) if (!func || !btf_type_is_func_proto(func)) return -EINVAL; - if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY)) { + if (func_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY | + KF_ITER_RET_INDEPENDENT)) { err = btf_check_iter_kfuncs(btf, func_name, func, func_flags); if (err) return err; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 3030beb531d9..7cf37690d41f 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -141,7 +141,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag mutex_init(&fp->aux->used_maps_mutex); mutex_init(&fp->aux->ext_mutex); mutex_init(&fp->aux->dst_mutex); - mutex_init(&fp->aux->st_ops_assoc_mutex); + spin_lock_init(&fp->aux->st_ops_assoc_lock); return fp; } @@ -291,7 +291,6 @@ void __bpf_prog_free(struct bpf_prog *fp) #endif mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); - mutex_destroy(&fp->aux->st_ops_assoc_mutex); kfree(fp->aux->poke_tab); kfree(fp->aux); } @@ -2640,8 +2639,7 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) /* * Whether a function has an insn that works with the arena base, which a JIT * sets up in the prologue when bpf_prog_uses_arena_insns() says so. Decided on - * the final insns of the function: an access through an arena pointer, or - * through a number in a program loaded with BPF_F_ARENA_SCALAR, has been + * the final insns of the function: an access through an arena pointer has been * rewritten to a BPF_PROBE_MEM32 form by then, a cast to the user address * space keeps BPF_ADDR_SPACE_CAST, and a kfunc call with an __arena argument, * which the JIT rebases, has a function model that says so. A load of an @@ -3519,12 +3517,6 @@ bool __weak bpf_jit_supports_arena(void) return false; } -/* Whether JIT takes BPF_REG_AX as the address of arena access */ -bool __weak bpf_jit_supports_arena_scalar(void) -{ - return false; -} - bool __weak bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) { return false; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index f062ef7dd943..41bc37e205e0 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -44,31 +44,6 @@ static bool is_addr_space_cast32(struct bpf_prog *prog, const struct bpf_insn *i return false; } -/* - * The insn accesses arena through a number. JITs add the base of arena - * to the register as it is, so the access goes through the low 32 bits of - * the number in BPF_REG_AX. Registers of the program are not changed: the - * number may be compared or stored later, and on another path the register - * may be PTR_TO_ARENA. - * - * Constant blinding leaves insns that use BPF_REG_AX alone, so the immediate - * of st through a number is not blinded. - */ -static int arena_scalar_access(const struct bpf_insn *insn, struct bpf_insn *buf) -{ - bool load = BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn); - struct bpf_insn *patch = buf; - - *patch++ = BPF_MOV32_REG(BPF_REG_AX, load ? insn->src_reg : insn->dst_reg); - *patch = *insn; - if (load) - patch->src_reg = BPF_REG_AX; - else - patch->dst_reg = BPF_REG_AX; - patch++; - return patch - buf; -} - /* Return the regno defined by the insn, or -1. */ static int insn_def_regno(const struct bpf_insn *insn) { @@ -1880,20 +1855,6 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */ insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code); - if (env->insn_aux_data[i + delta].arena_scalar) { - cnt = arena_scalar_access(insn, insn_buf); - - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - prog = new_prog; - env->prog = prog; - insn = prog->insnsi + i + delta; - goto next_insn; - } - /* Make sdiv/smod divide-by-minus-one exceptions impossible. */ if ((insn->code == (BPF_ALU64 | BPF_MOD | BPF_K) || insn->code == (BPF_ALU64 | BPF_DIV | BPF_K) || diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 648abe1ba63a..58e357302850 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1221,7 +1221,7 @@ static void bpf_async_refcount_put(struct bpf_async_cb *cb); static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) { struct bpf_hrtimer *t = container_of(hrtimer, struct bpf_hrtimer, timer); - struct bpf_map *map = t->cb.map; + struct bpf_map *map = READ_ONCE(t->cb.map); void *value = t->cb.value; bpf_callback_t callback_fn; void *key; @@ -1229,7 +1229,7 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) BTF_TYPE_EMIT(struct bpf_timer); callback_fn = rcu_dereference_check(t->cb.callback_fn, rcu_read_lock_bh_held()); - if (!callback_fn) + if (!callback_fn || !map) goto out; /* bpf_timer_cb() runs in hrtimer_run_softirq. It doesn't migrate and @@ -1254,7 +1254,7 @@ static void bpf_wq_work(struct work_struct *work) { struct bpf_work *w = container_of(work, struct bpf_work, work); struct bpf_async_cb *cb = &w->cb; - struct bpf_map *map = cb->map; + struct bpf_map *map; bpf_callback_t callback_fn; void *value = cb->value; void *key; @@ -1265,8 +1265,9 @@ static void bpf_wq_work(struct work_struct *work) /* keeps the program that callback_fn belongs to */ rcu_read_lock_trace(); + map = READ_ONCE(cb->map); callback_fn = READ_ONCE(cb->callback_fn); - if (!callback_fn) + if (!callback_fn || !map) goto out; key = map_key_from_value(map, value, &idx); @@ -1733,6 +1734,8 @@ static void bpf_async_cancel_and_free(struct bpf_async_kern *async) if (!cb) return; + /* Callbacks that run after this see NULL and return. */ + WRITE_ONCE(cb->map, NULL); bpf_async_update_prog_callback(cb, NULL, NULL); /* * No refcount_inc_not_zero(&cb->refcnt) here. Dropping the last diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index 5432c0fc442b..fa29d9b994e4 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -535,6 +535,14 @@ void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num) verbose(env, "%#llx", num); } +void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num) +{ + if (is_snum_decimal(num)) + verbose(env, "%d", num); + else + verbose(env, "%#x", (u32)num); +} + int tnum_strn(char *str, size_t size, struct tnum a) { /* print as a constant, if tnum is fully known */ diff --git a/kernel/bpf/scev.c b/kernel/bpf/scev.c index fea77772ca96..7b82fd79f9cc 100644 --- a/kernel/bpf/scev.c +++ b/kernel/bpf/scev.c @@ -1502,6 +1502,10 @@ static int simplify(struct scev *scev, u32 id, void *priv) return t; } + /* (zext32 (zext32 X)) -> (zext32 X) */ + if (is_zext32(scev, id, &a) && is_zext32(scev, a, &b)) + return a; + return id; } @@ -2024,7 +2028,8 @@ static void log_incompatible_scev_values(struct bpf_verifier_env *env, const cha } static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_state *st, - const struct scev_value *a, const struct scev_value *b) + const struct scev_value *a, const struct scev_value *b, + const struct linear_latch *latch) { const struct bpf_reg_state *ra, *rb; @@ -2035,7 +2040,7 @@ static bool scev_value_compatible(struct bpf_verifier_env *env, struct bpf_func_ } ra = scev_reg_for_read(env, st, a->ptr_reg); rb = scev_reg_for_read(env, st, b->ptr_reg); - if (ra && rb && bpf_same_memory_origin(ra, rb)) + if (ra && rb && bpf_same_memory_origin(ra, rb) && !latch->zext32 && !latch->jmp32) return true; incompatible: @@ -2267,7 +2272,7 @@ static bool compute_max_iters(struct bpf_verifier_env *env, } return false; } - if (!scev_value_compatible(env, st, &initial_val, &bound_val)) + if (!scev_value_compatible(env, st, &initial_val, &bound_val, latch)) return false; initial = initial_val.value; @@ -2306,11 +2311,15 @@ static bool compute_max_iters(struct bpf_verifier_env *env, if (!compute_backedge_taken_count(op, initial, step, bound, &backedge_taken_count)) return false; + /* + * max_header_count is u32 and U32_MAX means "infinite", + * check to avoid overflow below. + */ + if (backedge_taken_count >= U32_MAX - 1) + return false; + if (latch->zext32 || latch->jmp32) { - /* - * compute_backedge_taken_count() guarantees that 'total_change' - * does not overflow in 64-bit domain. - */ + /* total_change is guaranteed not to overflow at this point */ total_change = initial + step * backedge_taken_count; if (latch->jmp32 && is_signed_op(op)) { if ((s64)total_change < S32_MIN || (s64)total_change > S32_MAX) @@ -2321,13 +2330,6 @@ static bool compute_max_iters(struct bpf_verifier_env *env, } } - /* - * max_header_count is u32 and U32_MAX means "infinite", - * check to avoid overflow below. - */ - if (backedge_taken_count >= U32_MAX - 1) - return false; - /* The header executes once on entry and once for each taken backedge, hence +1. */ iters->max_header_count = backedge_taken_count + 1; /* @@ -2377,7 +2379,6 @@ static bool is_widenable_reg_type(const struct bpf_reg_state *reg) case PTR_TO_PACKET: case PTR_TO_PACKET_META: case PTR_TO_MEM: - case PTR_TO_BUF: case PTR_TO_BTF_ID: return true; default: @@ -2389,7 +2390,7 @@ static void scratch_widened_reg_id(struct bpf_verifier_env *env, struct bpf_reg_ { switch (base_type(reg->type)) { case SCALAR_VALUE: - reg->id = 0; + clear_scalar_id(reg); break; case PTR_TO_PACKET: case PTR_TO_PACKET_META: @@ -2454,11 +2455,13 @@ static void log_bounds(struct bpf_verifier_env *env, const struct bounds *bounds bpf_verbose_snum(env, cnum64_smin(bounds->r64)); bpf_log(log, ".."); bpf_verbose_snum(env, cnum64_smax(bounds->r64)); - if (bounds->r32.size < U32_MAX) { + if (bounds->r32.size < U32_MAX && + (cnum32_smin(bounds->r32) != cnum64_smin(bounds->r64) || + cnum32_smax(bounds->r32) != cnum64_smax(bounds->r64))) { bpf_log(log, " (s32 "); - bpf_verbose_snum(env, cnum32_smin(bounds->r32)); + bpf_verbose_snum32(env, cnum32_smin(bounds->r32)); bpf_log(log, ".."); - bpf_verbose_snum(env, cnum32_smax(bounds->r32)); + bpf_verbose_snum32(env, cnum32_smax(bounds->r32)); bpf_log(log, ")"); } bpf_log(log, " step %u", bounds->step); @@ -2700,6 +2703,9 @@ int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_sta reg = scev_reg_for_linear(cur_func, r); /* If SCEV for r is (linear ) */ if (reg && is_simple_linear(scev, r_scev, &coefs) && is_widenable_reg_type(reg)) { + /* 32-bit ALU operations on pointers convert them to scalars */ + if (coefs.zext32 && reg->type != SCALAR_VALUE) + goto cant_widen; /* Spills at varying offsets lose precision */ if (test_bit(r, loop->store_base_regs)) { spill_base = true; @@ -2891,13 +2897,15 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur is_unbound_scev_reg(env, entry_st, r)) continue; + /* + * A call in the loop can change the type: it invalidates packet pointers or + * releases a reference. The bounds are the offsets of the pointer at the entry. + * They say nothing about the value that is in the register now. + */ + if (reg->type != entry_reg->type) + continue; + linear_bounds(entry_reg, max_header_count, &coefs, &bounds); - bounds.r64 = cnum64_intersect(reg->r64, bounds.r64); - bounds.r32 = cnum32_intersect(reg->r32, bounds.r32); - if (cnum64_is_empty(bounds.r64)) { - verifier_bug(env, "scev clamp produced empty range for r%d", r); - return -EFAULT; - } if (log->level & BPF_LOG_LEVEL2) { bpf_log(log, "loop header at %d, clamping ", insn_idx); log_reg(env, r); @@ -2906,6 +2914,13 @@ int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *cur bpf_log(log, "\n"); } scratch_widened_reg_id(env, reg); + /* + * The range of a packet pointer goes together with its id and with the offsets + * that it had in the compare with pkt_end. Both are replaced here. What holds for + * the new offsets is the range at the entry. + */ + if (type_is_pkt_pointer(reg->type)) + reg->range = max(entry_reg->range, 0); err = bpf_set_reg_range(env, reg, bounds.r64, bounds.r32, bounds.base, bounds.step); if (err) diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index d7df6632a5c4..a6ae86f973e3 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -1358,13 +1358,33 @@ static bool need_loop_checkpoint(struct bpf_verifier_env *env, int insn_idx) return top->loop_id == insn_idx && top->terminates; } +/* Searches for an innermost loop entry starting from an innermost frame. */ static struct loop_stack_entry *current_loop(struct bpf_verifier_state *st) { - struct bpf_func_state *frame = st->frame[st->curframe]; + struct bpf_func_state *frame; + int fr; - if (frame->loop_stack_cnt == 0) - return NULL; - return &frame->loop_stack[frame->loop_stack_cnt - 1]; + for (fr = st->curframe; fr >= 0; fr--) { + frame = st->frame[fr]; + if (frame->loop_stack_cnt) + return &frame->loop_stack[frame->loop_stack_cnt - 1]; + } + return NULL; +} + +/* True if any of the loops call chain is currently in terminates. */ +static bool in_a_terminating_loop(struct bpf_verifier_state *st) +{ + struct bpf_func_state *frame; + int l, fr; + + for (fr = st->curframe; fr >= 0; fr--) { + frame = st->frame[fr]; + for (l = frame->loop_stack_cnt - 1; l >= 0; l--) + if (frame->loop_stack[l].terminates) + return true; + } + return false; } int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) @@ -1528,10 +1548,10 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx) */ old_loop = current_loop(&sl->state); cur_loop = current_loop(cur); - if (old_loop && cur_loop && - old_loop->terminates && cur_loop->terminates && - old_loop->entry_state == cur_loop->entry_state) { - if (states_equal(env, &sl->state, cur, RANGE_WITHIN)) { + if (old_loop && cur_loop && in_a_terminating_loop(cur)) { + if (old_loop->terminates && cur_loop->terminates && + old_loop->entry_state == cur_loop->entry_state && + states_equal(env, &sl->state, cur, RANGE_WITHIN)) { loop = true; goto hit; } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 90bc7230afa8..60c4d84b9aba 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2971,13 +2971,9 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_at BPF_F_XDP_HAS_FRAGS | BPF_F_XDP_DEV_BOUND_ONLY | BPF_F_TEST_REG_INVARIANTS | - BPF_F_ARENA_SCALAR | BPF_F_TOKEN_FD)) return -EINVAL; - if ((attr->prog_flags & BPF_F_ARENA_SCALAR) && !bpf_jit_supports_arena_scalar()) - return -EOPNOTSUPP; - bpf_prog_load_fixup_attach_type(attr); if (attr->prog_flags & BPF_F_TOKEN_FD) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5601d2056218..26a5ee0708a3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -211,6 +211,9 @@ static int acquire_reference(struct bpf_verifier_env *env, int insn_idx, int par static int __release_reference_nomark(struct bpf_verifier_state *state, int id); static int release_reference_nomark(struct bpf_verifier_env *env, int id); static int release_reference(struct bpf_verifier_env *env, int id); +static int invalidate_reference(struct bpf_verifier_env *env, int id); +static u32 reg_lifetime_id(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg); static void invalidate_non_owning_refs(struct bpf_verifier_env *env); static void invalidate_rcu_protected_refs(struct bpf_verifier_env *env); static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env); @@ -793,8 +796,8 @@ static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_ if (err) return err; - /* Track parent's id if the parent is a referenced object */ - parent_id = ref_obj->id; + /* All non-clone constructors take their backing object in R1. */ + parent_id = reg_lifetime_id(env, &cur_regs(env)[BPF_REG_1]); if (dynptr_type_referenced(type)) { int id; @@ -1074,7 +1077,7 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_reg_state *reg, int nr_slots) { struct bpf_func_state *state = bpf_func(env, reg); - int spi, i, j; + int spi, i, j, err; spi = iter_get_spi(env, reg, nr_slots); if (spi < 0) @@ -1084,8 +1087,11 @@ static int unmark_stack_slots_iter(struct bpf_verifier_env *env, struct bpf_stack_state *slot = bpf_stack_slot(state, spi - i); struct bpf_reg_state *st = &slot->spilled_ptr; - if (i == 0) - WARN_ON_ONCE(release_reference(env, st->id)); + if (i == 0) { + err = release_reference(env, st->id); + if (err) + return err; + } bpf_mark_reg_not_init(env, st); @@ -1627,6 +1633,12 @@ static bool reg_is_referenced(struct bpf_verifier_env *env, const struct bpf_reg return find_reference_state(env->cur_state, reg->id); } +static u32 reg_lifetime_id(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + return reg_is_referenced(env, reg) ? reg->id : reg->parent_id; +} + static int release_lock_state(struct bpf_verifier_env *env, int type, int id, void *ptr) { struct bpf_verifier_state *state = env->cur_state; @@ -2605,12 +2617,6 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, return &elem->st; } -static int cmp_subprogs(const void *a, const void *b) -{ - return ((struct bpf_subprog_info *)a)->start - - ((struct bpf_subprog_info *)b)->start; -} - /* Find subprogram that contains instruction at 'off' */ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off) { @@ -2645,26 +2651,29 @@ int bpf_find_subprog(struct bpf_verifier_env *env, int off) static int add_subprog(struct bpf_verifier_env *env, int off) { + struct bpf_subprog_info *subprog = env->subprog_info; int insn_cnt = env->prog->len; - int ret; + int i; if (off >= insn_cnt || off < 0) { verbose(env, "call to invalid destination\n"); return -EINVAL; } - ret = bpf_find_subprog(env, off); - if (ret >= 0) - return ret; + i = bpf_find_subprog(env, off); + if (i >= 0) + return i; if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { verbose(env, "too many subprograms\n"); return -E2BIG; } - /* determine subprog starts. The end is one before the next starts */ - env->subprog_info[env->subprog_cnt].exit_idx = U32_MAX; - env->subprog_info[env->subprog_cnt++].start = off; - sort(env->subprog_info, env->subprog_cnt, - sizeof(env->subprog_info[0]), cmp_subprogs, NULL); - return env->subprog_cnt - 1; + /* + * Determine subprog starts. The end is one before the next starts. + * The array stays sorted: subprogs that start after 'off' move up. + */ + for (i = env->subprog_cnt++; i > 0 && subprog[i - 1].start > off; i--) + subprog[i] = subprog[i - 1]; + subprog[i] = (struct bpf_subprog_info){ .start = off, .exit_idx = U32_MAX }; + return i; } static int bpf_find_exception_callback_insn_off(struct bpf_verifier_env *env) @@ -3812,12 +3821,6 @@ static bool __is_pointer_value(bool allow_ptr_leaks, return is_pointer_regtype(reg->type); } -static void clear_scalar_id(struct bpf_reg_state *reg) -{ - reg->id = 0; - reg->delta = 0; -} - static void assign_scalar_id_before_mov(struct bpf_verifier_env *env, struct bpf_reg_state *src_reg) { @@ -4654,7 +4657,12 @@ static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_st } caller = vstate->frame[vstate->curframe - 1]; - arg = &caller->stack_arg_regs[spi]; + arg = bpf_get_spilled_stack_arg(spi, caller); + if (!arg) { + verbose(env, "invalid read from uninitialized stack arg off %d\n", off); + return -EACCES; + } + cur = vstate->frame[vstate->curframe]; bpf_diag_mod_begin(env, &cur->regs[dst_regno], arg, BPF_DIAG_MOD_WRITE); cur->regs[dst_regno] = *arg; @@ -5483,19 +5491,6 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } -/* - * There are no address spaces in Rust, addresses of arena are plain numbers. - * When the program is loaded with BPF_F_ARENA_SCALAR and has an arena a load or - * a store through a number is an access to arena at the low 32 bits of it, - * like the access through PTR_TO_ARENA is. The register stays a number. - */ -static bool is_arena_scalar(struct bpf_verifier_env *env, int regno) -{ - const struct bpf_reg_state *reg = reg_state(env, regno); - - return reg->type == SCALAR_VALUE && env->arena_scalar && env->prog->aux->arena; -} - static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, struct bpf_insn *insn) { @@ -5526,7 +5521,7 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_sk_reg(env, regno)) return false; - if (is_arena_reg(env, regno) || is_arena_scalar(env, regno)) + if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); if (is_load_acq_unsafe(env, regno, insn)) return false; @@ -6848,9 +6843,14 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, flag &= ~PTR_MAYBE_NULL; if (atype == BPF_READ && value_regno >= 0) { + u32 parent_id = reg_lifetime_id(env, reg); + ret = mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag); if (ret < 0) return ret; + if ((regs[value_regno].type & PTR_TRUSTED) && + !(regs[value_regno].type & (MEM_RCU | MEM_PERCPU | MEM_USER))) + regs[value_regno].parent_id = parent_id; } return 0; @@ -7502,22 +7502,6 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type, bool allow_trust_mismatch); -/* - * Returns the register to check the access of the current insn with. - * For a number in a program with an arena that is 'arena'. - */ -static struct bpf_reg_state *mem_access_reg(struct bpf_verifier_env *env, int regno, - struct bpf_reg_state *arena) -{ - if (!is_arena_scalar(env, regno)) - return cur_regs(env) + regno; - - memset(arena, 0, sizeof(*arena)); - arena->type = PTR_TO_ARENA; - env->insn_aux_data[env->insn_idx].arena_scalar = true; - return arena; -} - static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, bool strict_alignment_once, bool is_ldsx, bool allow_trust_mismatch, const char *ctx) @@ -7525,7 +7509,6 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state arena, *src_reg; enum bpf_reg_type src_reg_type; int err; @@ -7547,16 +7530,16 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - src_reg = mem_access_reg(env, insn->src_reg, &arena); - src_reg_type = src_reg->type; + src_reg_type = regs[insn->src_reg].type; /* * check_stack_read_fixed_off() may refine the modification's origin to * the source stack slot. */ bpf_diag_mod_begin(env, ®s[insn->dst_reg], NULL, BPF_DIAG_MOD_WRITE); - err = check_mem_access(env, env->insn_idx, src_reg, argno_from_reg(insn->src_reg), - insn->off, BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, + err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, + argno_from_reg(insn->src_reg), insn->off, + BPF_SIZE(insn->code), BPF_READ, insn->dst_reg, strict_alignment_once, is_ldsx); /* * Paths through a kernel struct pointer and through other pointers @@ -7584,7 +7567,6 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_verifier_state *vstate = env->cur_state; struct bpf_func_state *state = vstate->frame[vstate->curframe]; struct bpf_reg_state *regs = cur_regs(env); - struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; int err; @@ -7606,12 +7588,12 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err) return err; - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - dst_reg_type = dst_reg->type; + dst_reg_type = regs[insn->dst_reg].type; /* Check if (dst_reg + off) is writeable. */ - err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), - insn->off, BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, + err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, + argno_from_reg(insn->dst_reg), insn->off, + BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg, strict_alignment_once, false); err = err ?: save_aux_ptr_type(env, dst_reg_type, false); @@ -7621,7 +7603,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn, static int check_atomic_rmw(struct bpf_verifier_env *env, struct bpf_insn *insn) { - struct bpf_reg_state arena, *dst_reg; + struct bpf_reg_state *dst_reg; int load_reg; int err; @@ -7666,9 +7648,6 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return -EACCES; } - /* load_reg may be dst_reg. Look at dst_reg before it's marked as unknown. */ - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - load_reg = bpf_atomic_load_reg(insn); if (load_reg >= 0) { /* check and record load of old value */ @@ -7677,6 +7656,8 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, return err; } + dst_reg = cur_regs(env) + insn->dst_reg; + /* Check whether we can read the memory, with second call for fetch * case to simulate the register fill. */ @@ -8636,6 +8617,21 @@ static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_ITER_DESTROY; } +static bool iter_next_borrows_struct(struct bpf_call_arg_meta *meta) +{ + const struct btf_type *t; + + if (!(meta->kfunc_flags & KF_ITER_NEXT) || + (meta->kfunc_flags & (KF_ITER_RET_INDEPENDENT | KF_ACQUIRE))) + return false; + + t = btf_type_skip_modifiers(meta->btf, meta->func_proto->type, NULL); + if (!t || !btf_type_is_ptr(t)) + return false; + t = btf_type_skip_modifiers(meta->btf, t->type, NULL); + return t && __btf_type_is_struct(t); +} + static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx, const struct btf_param *arg) { @@ -8746,6 +8742,13 @@ static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state * meta->iter.frameno = reg->frameno; update_ref_obj(&meta->ref_obj, &bpf_stack_slot(state, spi)->spilled_ptr); + /* The iterator stays. What was borrowed from the previous element goes. */ + if (iter_next_borrows_struct(meta)) { + err = invalidate_reference(env, meta->ref_obj.id); + if (err) + return err; + } + if (is_iter_destroy_kfunc(meta)) { err = unmark_stack_slots_iter(env, reg, nr_slots); if (err) @@ -11073,8 +11076,8 @@ static int idstack_pop(struct bpf_idmap *idmap) return idmap->map[--idmap->cnt].old; } -/* Release id and objects derived from it iteratively in a DFS manner */ -static int release_reference(struct bpf_verifier_env *env, int id) +/* Invalidate id and objects derived from it iteratively in a DFS manner */ +static int invalidate_reference(struct bpf_verifier_env *env, int id) { u32 mask = (1 << STACK_SPILL) | (1 << STACK_DYNPTR); struct bpf_verifier_state *vstate = env->cur_state; @@ -11089,11 +11092,6 @@ static int release_reference(struct bpf_verifier_env *env, int id) if (err) return err; - if (find_reference_state(vstate, id)) { - err = release_reference_nomark(env, id); - WARN_ON_ONCE(err); - } - while ((id = idstack_pop(idstack))) { /* * Child references are inaccessible after parent is released, @@ -11145,6 +11143,14 @@ static int release_reference(struct bpf_verifier_env *env, int id) return 0; } +static int release_reference(struct bpf_verifier_env *env, int id) +{ + if (find_reference_state(env->cur_state, id)) + WARN_ON_ONCE(release_reference_nomark(env, id)); + + return invalidate_reference(env, id); +} + static void invalidate_non_owning_refs(struct bpf_verifier_env *env) { struct bpf_func_state *unused; @@ -11409,6 +11415,10 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins err = btf_check_subprog_call(env, subprog); if (err == -EFAULT) return err; + if (bpf_in_stack_arg_cnt(&env->subprog_info[subprog])) { + verbose(env, "callback subprog cannot have stack args\n"); + return -EINVAL; + } /* set_callee_state is used for direct subprog calls, but we are * interested in validating only BPF helpers that can call subprogs as @@ -15822,6 +15832,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, regs[BPF_REG_0].btf = desc_btf; regs[BPF_REG_0].type = type; regs[BPF_REG_0].btf_id = ptr_type_id; + if (iter_next_borrows_struct(&meta) && !(type & MEM_RCU)) + regs[BPF_REG_0].parent_id = meta.ref_obj.id; } if (is_kfunc_ret_null(&meta)) { @@ -17274,11 +17286,14 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn, } } +static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct bpf_reg_state *dst_reg, struct bpf_reg_state src_reg); + static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct bpf_reg_state *dst_reg) + struct bpf_reg_state *dst_reg, const struct bpf_reg_state *src_reg) { struct bpf_verifier_state *branch; - struct bpf_reg_state *regs; + struct bpf_reg_state *reg; bool alu32; if (reg_smin(dst_reg) == -1 && reg_smax(dst_reg) == 0) @@ -17288,19 +17303,25 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins else return 0; - branch = push_stack(env, env->insn_idx, env->insn_idx, false); + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) return PTR_ERR(branch); - regs = branch->frame[branch->curframe]->regs; + reg = &branch->frame[branch->curframe]->regs[insn->dst_reg]; if (alu32) { - __mark_reg32_known(®s[insn->dst_reg], 0); + __mark_reg32_known(reg, 0); __mark_reg32_known(dst_reg, -1ull); } else { - __mark_reg_known(®s[insn->dst_reg], 0); + __mark_reg_known(reg, 0); __mark_reg_known(dst_reg, -1ull); } - return 0; + /* + * The insn is done with in the branch too, and what gets to the next insn is a state + * like any other. At this insn the branch is a part of the state that is looked at: + * it must not be compared with it. + */ + clear_scalar_id(reg); + return adjust_scalar_min_max_vals(env, insn, reg, *src_reg); } /* WARNING: This function does calculations on 64-bit values, but the actual @@ -17405,7 +17426,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, break; case BPF_AND: if (tnum_is_const(src_reg.var_off)) { - ret = maybe_fork_scalars(env, insn, dst_reg); + ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg); if (ret) return ret; } @@ -17416,7 +17437,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, break; case BPF_OR: if (tnum_is_const(src_reg.var_off)) { - ret = maybe_fork_scalars(env, insn, dst_reg); + ret = maybe_fork_scalars(env, insn, dst_reg, &src_reg); if (ret) return ret; } @@ -20236,17 +20257,16 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state) return check_stack_arg_write(env, state, insn->off, NULL); } - struct bpf_reg_state arena, *dst_reg; enum bpf_reg_type dst_reg_type; err = check_reg_arg(env, insn->dst_reg, SRC_OP); if (err) return err; - dst_reg = mem_access_reg(env, insn->dst_reg, &arena); - dst_reg_type = dst_reg->type; + dst_reg_type = cur_regs(env)[insn->dst_reg].type; - err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), + err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, + argno_from_reg(insn->dst_reg), insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, false, false); if (err) @@ -23882,7 +23902,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (is_priv) env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ; env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS; - env->arena_scalar = attr->prog_flags & BPF_F_ARENA_SCALAR; env->explored_states = kvzalloc_objs(struct list_head, state_htab_size(env), diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index dfac5e1b8f40..14adfb85fdb2 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -867,15 +867,12 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struc siginfo = &info; } - /* Similar to bpf_probe_write_user, task needs to be - * in a sound condition and kernel memory access be - * permitted in order to send signal to the current - * task. + /* + * Similar to bpf_probe_write_user, task needs to be + * in a sound condition. */ if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING))) return -EPERM; - if (unlikely(!nmi_uaccess_okay())) - return -EPERM; /* Task should not be pid=1 to avoid kernel panic. */ if (unlikely(is_global_init(task))) return -EPERM; diff --git a/net/core/filter.c b/net/core/filter.c index b7ac145b68b1..09c60a58c82a 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1789,7 +1789,7 @@ BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset, if (unlikely(!ptr)) goto err_clear; if (ptr != to) - memcpy(to, ptr, len); + memmove(to, ptr, len); return 0; err_clear: @@ -1871,7 +1871,7 @@ BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb, ptr = start + offset; if (likely(ptr + len <= end)) { - memcpy(to, ptr, len); + memmove(to, ptr, len); return 0; } @@ -4298,7 +4298,7 @@ BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset, if (!ptr) bpf_xdp_copy_buf(xdp, offset, buf, len, false); else - memcpy(buf, ptr, len); + memmove(buf, ptr, len); return 0; } diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index f29b0aad440c..8d7228834305 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -1357,12 +1357,6 @@ enum bpf_perf_event_type { /* The verifier internal test flag. Behavior is undefined */ #define BPF_F_TEST_REG_INVARIANTS (1U << 7) -/* - * Load and store through a number is an access to the arena of the program - * at the low 32 bits of the number. It's for programs written in Rust. - */ -#define BPF_F_ARENA_SCALAR (1U << 8) - /* link_create.kprobe_multi.flags used in LINK_CREATE command for * BPF_TRACE_KPROBE_MULTI attach type to create return probe. */ diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index b37b727ef216..9d160b5b9c0e 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h @@ -248,21 +248,13 @@ enum libbpf_tristate { #define ___bpf_fill(arr, args...) \ ___bpf_apply(___bpf_fill, ___bpf_narg(args))(arr, 0, args) -/* - * Format strings are in a section of their own. When global data of the object - * is in arena (the object has .arena.data section) libbpf keeps .rodata.str* - * sections in read-only maps, which is where helpers and kfuncs take strings - * from. - */ -#define ___bpf_fmt_sec __attribute__((section(".rodata.str"))) - /* * BPF_SEQ_PRINTF to wrap bpf_seq_printf to-be-printed values * in a structure. */ #define BPF_SEQ_PRINTF(seq, fmt, args...) \ ({ \ - static const char ___fmt[] ___bpf_fmt_sec = fmt; \ + static const char ___fmt[] = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -280,7 +272,7 @@ enum libbpf_tristate { */ #define BPF_SNPRINTF(out, out_size, fmt, args...) \ ({ \ - static const char ___fmt[] ___bpf_fmt_sec = fmt; \ + static const char ___fmt[] = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -295,7 +287,7 @@ enum libbpf_tristate { #ifdef BPF_NO_GLOBAL_DATA #define BPF_PRINTK_FMT_MOD #else -#define BPF_PRINTK_FMT_MOD static const ___bpf_fmt_sec +#define BPF_PRINTK_FMT_MOD static const #endif #define __bpf_printk(fmt, ...) \ @@ -311,7 +303,7 @@ enum libbpf_tristate { */ #define __bpf_vprintk(fmt, args...) \ ({ \ - static const char ___fmt[] ___bpf_fmt_sec = fmt; \ + static const char ___fmt[] = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ @@ -325,7 +317,7 @@ enum libbpf_tristate { #define bpf_stream_printk(stream_id, fmt, args...) \ ({ \ - static const char ___fmt[] ___bpf_fmt_sec = fmt; \ + static const char ___fmt[] = fmt; \ unsigned long long ___param[___bpf_narg(args)]; \ \ _Pragma("GCC diagnostic push") \ diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 8fcb8cf39cc0..fb8a1f07c5b6 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -552,26 +552,6 @@ struct bpf_struct_ops { #define STRUCT_OPS_SEC ".struct_ops" #define STRUCT_OPS_LINK_SEC ".struct_ops.link" #define ARENA_SEC ".addr_space.1" -/* - * An object with this section keeps its global data in arena instead of - * array maps. What is in the section doesn't matter. .data, .bss and .rodata - * sections become a part of the arena map of the object, like __arena global - * variables are. If the object doesn't declare an arena map libbpf creates - * one that is just large enough for the data. bpf_object__find_map_by_name() - * finds it by the name "arena" and bpf_map__set_max_entries() changes its - * size before the object is loaded. - * Read-only sections with pointers to functions and sections with constant - * strings stay frozen array maps. A constant that a helper or a kfunc takes - * by pointer has to be in such section. - * Pointers to data that are stored in data become addresses of arena. - * The load fails if such pointer points to a section that is not in arena, - * or if the arena is pinned or reused. - * - * It's for programs written in Rust. There are no address spaces in Rust, - * the program accesses arena through plain numbers, so programs of the object - * are loaded with BPF_F_ARENA_SCALAR. - */ -#define ARENA_DATA_SEC ".arena.data" enum libbpf_map_type { LIBBPF_MAP_UNSPEC, @@ -627,13 +607,6 @@ struct bpf_map { /* pointers to functions in the data of an internal map, see obj->func_ptrs */ struct func_ptr *func_ptrs; size_t func_ptr_cnt; - /* - * Data of the internal map is a part of arena data at arena_off. - * The map is not created, unless it's a copy that is in arena. - */ - bool in_arena; - bool arena_copy; - size_t arena_off; }; enum extern_type { @@ -808,17 +781,6 @@ struct bpf_object { void *arena_data; size_t arena_data_sz; size_t arena_data_off; - bool data_in_arena; - bool arena_mapped; - /* pointers to data in the data of internal maps, when data is in arena */ - struct data_ptr { - int sec_idx; - size_t sec_off; - int targ_sec_idx; - size_t targ_off; - char *sym_name; - } *data_ptrs; - size_t data_ptr_cnt; void *jumptables_data; size_t jumptables_data_sz; @@ -1644,7 +1606,6 @@ static struct bpf_object *bpf_object__new(const char *path, obj->efile.obj_buf = obj_buf; obj->efile.obj_buf_sz = obj_buf_sz; obj->efile.btf_maps_shndx = -1; - obj->efile.arena_data_shndx = -1; obj->kconfig_map_idx = -1; obj->arena_map_idx = -1; @@ -3126,136 +3087,6 @@ static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map, return 0; } -static bool map_is_const_str(const struct bpf_map *map) -{ - return map->libbpf_type == LIBBPF_MAP_RODATA && - str_has_pfx(map->real_name, RODATA_SEC ".str"); -} - -static bool map_data_goes_to_arena(const struct bpf_object *obj, const struct bpf_map *map) -{ - int i; - - switch (map->libbpf_type) { - case LIBBPF_MAP_DATA: - case LIBBPF_MAP_BSS: - return true; - case LIBBPF_MAP_RODATA: - break; - default: - return false; - } - - /* - * Const strings that kfuncs and helpers take have to be in a frozen - * map. Pointers to them that are stored in data point to a copy of - * the strings that is in arena. - */ - if (map_is_const_str(map)) - return true; - /* - * Read-only data with pointers stays a frozen map: the kernel - * recognizes pointers to functions in it. - */ - if (str_has_pfx(map->real_name, DATA_REL_RO_SEC)) - return false; - for (i = 0; i < obj->efile.sec_cnt; i++) { - const struct elf_sec_desc *sec = &obj->efile.secs[i]; - - if (sec->sec_type == SEC_RELO && sec->shdr->sh_info == map->sec_idx) - return false; - } - return true; -} - -/* Returns the offset of the data of the map in arena data that is sz bytes so far */ -static size_t map_arena_off(const struct bpf_object *obj, const struct bpf_map *map, size_t sz) -{ - size_t align = obj->efile.secs[map->sec_idx].shdr->sh_addralign; - - return roundup(sz, max(align, sizeof(__u64))); -} - -/* - * Make .data, .bss and .rodata a part of arena data. They follow __arena - * variables of the object, if there are any. - */ -static int bpf_object__init_arena_data(struct bpf_object *obj) -{ - const size_t page_sz = sysconf(_SC_PAGE_SIZE); - size_t sz = obj->arena_data_sz; - struct bpf_map *map; - void *data; - int i; - - if (!obj->data_in_arena) - return 0; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - if (map_data_goes_to_arena(obj, map)) - sz = map_arena_off(obj, map, sz) + map->def.value_size; - } - if (sz == obj->arena_data_sz) - return 0; - - if (obj->arena_map_idx < 0) { - /* bpf_object__init_user_btf_maps() didn't look: there is no .maps section */ - if (obj->efile.arena_data) { - pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n", - ARENA_SEC); - return -ENOENT; - } - - map = bpf_object__add_map(obj); - if (IS_ERR(map)) - return PTR_ERR(map); - - map->real_name = strdup("arena"); - map->name = strdup("arena"); - if (!map->real_name || !map->name) { - zfree(&map->real_name); - zfree(&map->name); - return -ENOMEM; - } - map->sec_idx = -1; - map->def.type = BPF_MAP_TYPE_ARENA; - map->def.max_entries = roundup(sz, page_sz) / page_sz; - map->def.map_flags = BPF_F_MMAPABLE; - obj->arena_map_idx = map - obj->maps; - } - - data = realloc(obj->arena_data, sz); - if (!data) - return -ENOMEM; - memset(data + obj->arena_data_sz, 0, sz - obj->arena_data_sz); - sz = obj->arena_data_sz; - obj->arena_data = data; - obj->maps[obj->arena_map_idx].mmaped = data; - - for (i = 0; i < obj->nr_maps; i++) { - map = &obj->maps[i]; - if (!map_data_goes_to_arena(obj, map)) - continue; - sz = map_arena_off(obj, map, sz); - map->arena_off = sz; - memcpy(data + sz, map->mmaped, map->def.value_size); - sz += map->def.value_size; - pr_debug("map '%s': data is in arena at offset %zu\n", map->name, map->arena_off); - if (map_is_const_str(map)) { - map->arena_copy = true; - continue; - } - munmap(map->mmaped, bpf_map_mmap_sz(map)); - /* make bpf_map__initial_value() and bpf_map__set_initial_value() work */ - map->mmaped = data + map->arena_off; - map->autocreate = false; - map->in_arena = true; - } - obj->arena_data_sz = sz; - return 0; -} - static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, const char *pin_root_path) { @@ -3348,7 +3179,6 @@ static int bpf_object__init_maps(struct bpf_object *obj, err = err ?: bpf_object__init_global_data_maps(obj); err = err ?: bpf_object__init_kconfig_map(obj); err = err ?: bpf_object_init_struct_ops(obj); - err = err ?: bpf_object__init_arena_data(obj); return err; } @@ -4163,11 +3993,6 @@ static int bpf_object__elf_collect(struct bpf_object *obj) if (!sh) return -LIBBPF_ERRNO__FORMAT; - /* the second pass has to know it when it sees relocations of data */ - name = elf_sec_str(obj, sh->sh_name); - if (name && strcmp(name, ARENA_DATA_SEC) == 0) - obj->data_in_arena = true; - if (sh->sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { pr_warn("elf: multiple symbol tables in %s\n", obj->path); @@ -4284,8 +4109,6 @@ static int bpf_object__elf_collect(struct bpf_object *obj) } else if (strcmp(name, ARENA_SEC) == 0) { obj->efile.arena_data = data; obj->efile.arena_data_shndx = idx; - } else if (strcmp(name, ARENA_DATA_SEC) == 0) { - /* the marker, see the first pass */ } else if (strcmp(name, JUMPTABLES_SEC) == 0) { obj->jumptables_data = malloc(data->d_size); if (!obj->jumptables_data) @@ -4310,9 +4133,6 @@ static int bpf_object__elf_collect(struct bpf_object *obj) * have pointers to functions. */ if (!section_have_execinstr(obj, targ_sec_idx) && - !(obj->data_in_arena && - (!strcmp(name, ".rel" DATA_SEC) || - str_has_pfx(name, ".rel" DATA_SEC "."))) && strcmp(name, ".rel" RODATA_SEC) && !str_has_pfx(name, ".rel" RODATA_SEC ".") && strcmp(name, ".rel" DATA_REL_RO_SEC) && @@ -5064,10 +4884,6 @@ static int bpf_program__record_reloc(struct bpf_program *prog, reloc_desc->insn_idx = insn_idx; reloc_desc->map_idx = map_idx; reloc_desc->sym_off = sym->st_value; - if (map->in_arena) { - reloc_desc->map_idx = obj->arena_map_idx; - reloc_desc->sym_off += map->arena_off; - } return 0; } @@ -5286,9 +5102,6 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) if (map_is_created(map)) return libbpf_err(-EBUSY); - if (map->in_arena) - return libbpf_err(-EOPNOTSUPP); - map->autocreate = autocreate; return 0; } @@ -5569,45 +5382,6 @@ bpf_object__reuse_map(struct bpf_map *map) return 0; } -static struct bpf_map *bpf_object__sec_map(struct bpf_object *obj, int sec_idx) -{ - int i; - - for (i = 0; i < obj->nr_maps; i++) - if (bpf_map__is_internal(&obj->maps[i]) && obj->maps[i].sec_idx == sec_idx) - return &obj->maps[i]; - return NULL; -} - -static int bpf_object__mmap_arena(struct bpf_object *obj, struct bpf_map *map) -{ - int i, err; - - map->mmaped = mmap((void *)(long)map->map_extra, - bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, - map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, - map->fd, 0); - if (map->mmaped == MAP_FAILED) { - err = -errno; - map->mmaped = NULL; - pr_warn("map '%s': failed to mmap arena: %s\n", - map->name, errstr(err)); - return err; - } - if (obj->arena_data) { - memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, - obj->arena_data_sz); - zfree(&obj->arena_data); - } - for (i = 0; i < obj->nr_maps; i++) { - struct bpf_map *m = &obj->maps[i]; - - if (m->in_arena) - m->mmaped = map->mmaped + obj->arena_data_off + m->arena_off; - } - return 0; -} - static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { @@ -5968,10 +5742,6 @@ bpf_object__create_maps(struct bpf_object *obj) continue; } - /* see bpf_object__relocate_data_ptrs() */ - if (map->def.type == BPF_MAP_TYPE_ARENA && obj->arena_mapped) - continue; - err = map_set_def_max_entries(map); if (err) goto err_out; @@ -6009,9 +5779,22 @@ bpf_object__create_maps(struct bpf_object *obj) if (err < 0) goto err_out; } else if (map->def.type == BPF_MAP_TYPE_ARENA) { - err = bpf_object__mmap_arena(obj, map); - if (err) + map->mmaped = mmap((void *)(long)map->map_extra, + bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, + map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED, + map->fd, 0); + if (map->mmaped == MAP_FAILED) { + err = -errno; + map->mmaped = NULL; + pr_warn("map '%s': failed to mmap arena: %s\n", + map->name, errstr(err)); return err; + } + if (obj->arena_data) { + memcpy(map->mmaped + obj->arena_data_off, obj->arena_data, + obj->arena_data_sz); + zfree(&obj->arena_data); + } } if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { err = init_map_in_map_slots(obj, map); @@ -8026,81 +7809,6 @@ static int bpf_program_fixup_func_info(struct bpf_object *obj, struct bpf_progra return err; } -/* - * Turn pointers to data that are stored in data into addresses of arena. - * The arena is created here, ahead of the other maps: the address has to be - * known before the maps that hold the pointers are created. - */ -static int bpf_object__relocate_data_ptrs(struct bpf_object *obj) -{ - struct bpf_map *arena, *map, *targ; - __u64 addr, val; - size_t i; - int err; - - if (!obj->data_ptr_cnt) - return 0; - - for (i = 0; i < obj->data_ptr_cnt; i++) { - struct data_ptr *p = &obj->data_ptrs[i]; - - if (p->targ_sec_idx < 0) - continue; - map = bpf_object__sec_map(obj, p->sec_idx); - targ = bpf_object__sec_map(obj, p->targ_sec_idx); - if (map && (map->in_arena || map->autocreate) && - (!targ || (!targ->in_arena && !targ->arena_copy))) { - pr_warn("sec '%s': pointer to '%s' at offset %zu can't be resolved: sec '%s' is not in arena\n", - map->real_name, p->sym_name, p->sec_off, - targ ? targ->real_name : ""); - return -LIBBPF_ERRNO__RELOC; - } - } - - if (obj->gen_loader) { - pr_warn("pointers to data in data are not supported by light skeleton\n"); - return -ENOTSUP; - } - - if (obj->arena_map_idx < 0) - return 0; - - arena = &obj->maps[obj->arena_map_idx]; - if (!arena->autocreate) - return 0; - /* libbpf doesn't mmap an arena that it didn't create, so the address is not known */ - if (arena->reused || arena->pin_path) { - pr_warn("map '%s': pointers to data in data are not supported with pinned or reused arena\n", - arena->name); - return -ENOTSUP; - } - - err = bpf_object__create_map(obj, arena, false); - err = err ?: bpf_object__mmap_arena(obj, arena); - if (err) { - pr_warn("map '%s': failed to create: %s\n", arena->name, errstr(err)); - return err; - } - obj->arena_mapped = true; - - for (i = 0; i < obj->data_ptr_cnt; i++) { - struct data_ptr *p = &obj->data_ptrs[i]; - - map = bpf_object__sec_map(obj, p->sec_idx); - if (!map || (!map->in_arena && !map->autocreate)) - continue; - - addr = (__u64)(unsigned long)arena->mmaped + obj->arena_data_off + p->targ_off; - if (p->targ_sec_idx >= 0) - addr += bpf_object__sec_map(obj, p->targ_sec_idx)->arena_off; - - memcpy(&val, map->mmaped + p->sec_off, sizeof(val)); - val += addr; - memcpy(map->mmaped + p->sec_off, &val, sizeof(val)); - } - return 0; -} - static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) { struct bpf_program *prog; @@ -8123,10 +7831,8 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat size_t mmap_sz = bpf_map_mmap_sz(arena_map); if (data_sz > mmap_sz) { - pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global %s of size %zu\n", - arena_map->name, mmap_sz, - obj->data_in_arena ? "data" : "__arena variables", - obj->arena_data_sz); + pr_warn("map '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n", + arena_map->name, mmap_sz, obj->arena_data_sz); return -E2BIG; } @@ -8135,10 +7841,6 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat obj->arena_data_off = mmap_sz - data_sz; } - err = bpf_object__relocate_data_ptrs(obj); - if (err) - return err; - /* Before relocating calls pre-process relocations and mark * few ld_imm64 instructions that points to subprogs. * Otherwise bpf_object__reloc_code() later would have to consider @@ -8371,85 +8073,12 @@ static int bpf_object__collect_map_relos(struct bpf_object *obj, return 0; } -/* - * A pointer to data that is stored in data. When data is in arena - * the program dereferences what it loads from data, so the pointer has - * to be the address of the target in arena. - */ -static int bpf_object__collect_data_ptr(struct bpf_object *obj, const char *relo_sec_name, - int relo_idx, size_t sec_idx, const Elf64_Rel *rel, - const Elf64_Sym *sym) -{ - Elf_Data *scn_data = obj->efile.secs[sec_idx].data; - const char *sym_name = elf_sym_str(obj, sym->st_name) ?: ""; - struct data_ptr *ptrs; - - if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_shndx < obj->efile.sec_cnt) - sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)) ?: ""; - - if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || - sym->st_shndx >= obj->efile.sec_cnt || - (sym->st_shndx != obj->efile.arena_data_shndx && - !bpf_object__shndx_is_data(obj, sym->st_shndx)) || - rel->r_offset >= scn_data->d_size || - scn_data->d_size - rel->r_offset < sizeof(__u64)) { - pr_warn("sec '%s': relo #%d: can't resolve pointer to '%s' at offset %zu when data is in arena\n", - relo_sec_name, relo_idx, sym_name, (size_t)rel->r_offset); - return -LIBBPF_ERRNO__RELOC; - } - - ptrs = libbpf_reallocarray(obj->data_ptrs, obj->data_ptr_cnt + 1, sizeof(*ptrs)); - if (!ptrs) - return -ENOMEM; - obj->data_ptrs = ptrs; - - ptrs[obj->data_ptr_cnt].sec_idx = sec_idx; - ptrs[obj->data_ptr_cnt].sec_off = rel->r_offset; - /* __arena variables are at the start of arena data */ - ptrs[obj->data_ptr_cnt].targ_sec_idx = - sym->st_shndx == obj->efile.arena_data_shndx ? -1 : sym->st_shndx; - ptrs[obj->data_ptr_cnt].targ_off = sym->st_value; - ptrs[obj->data_ptr_cnt].sym_name = strdup(sym_name); - if (!ptrs[obj->data_ptr_cnt].sym_name) - return -ENOMEM; - obj->data_ptr_cnt++; - - pr_debug("sec '%s': relo #%d: pointer at offset %zu to '%s'\n", - relo_sec_name, relo_idx, (size_t)rel->r_offset, sym_name); - return 0; -} - -/* Collect pointers in a data section that went to arena */ -static int bpf_object__collect_data_relos(struct bpf_object *obj, - Elf64_Shdr *shdr, Elf_Data *data) -{ - int i, err, nrels = shdr->sh_size / shdr->sh_entsize; - const char *relo_sec_name; - Elf64_Sym *sym; - Elf64_Rel *rel; - - relo_sec_name = elf_sec_str(obj, shdr->sh_name) ?: ""; - for (i = 0; i < nrels; i++) { - rel = elf_rel_by_idx(data, i); - sym = rel ? elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)) : NULL; - if (!sym) { - pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); - return -LIBBPF_ERRNO__FORMAT; - } - err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, shdr->sh_info, rel, sym); - if (err) - return err; - } - return 0; -} - /* * Collect pointers to functions in a read-only data section. They are * R_BPF_64_ABS64 relocations against .text section, where the offset of * a static function in the section is stored in place. Relocations in data * sections were ignored before pointers to functions were supported. Those - * that are something else, e.g. pointers to data, still are, unless data - * is in arena. + * that are something else, e.g. pointers to data, still are. */ static int bpf_object__collect_rodata_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) @@ -8485,15 +8114,6 @@ static int bpf_object__collect_rodata_relos(struct bpf_object *obj, if (ELF64_R_TYPE(rel->r_info) != R_BPF_64_ABS64 || !sym_is_subprog(sym, obj->efile.text_shndx)) { - int err; - - if (obj->data_in_arena) { - err = bpf_object__collect_data_ptr(obj, relo_sec_name, i, - sec_idx, rel, sym); - if (err) - return err; - continue; - } pr_debug("sec '%s': relo #%d: not a pointer to a function, skipping...\n", relo_sec_name, i); continue; @@ -8569,8 +8189,6 @@ static int bpf_object__collect_relos(struct bpf_object *obj) if (obj->efile.secs[idx].sec_type == SEC_RODATA) err = bpf_object__collect_rodata_relos(obj, shdr, data); - else if (obj->efile.secs[idx].sec_type == SEC_DATA) - err = bpf_object__collect_data_relos(obj, shdr, data); else if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) err = bpf_object__collect_st_ops_relos(obj, shdr, data); else if (idx == obj->efile.btf_maps_shndx) @@ -8914,9 +8532,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog } load_attr.log_level = log_level; load_attr.prog_flags = prog->prog_flags; - /* the program accesses its data in arena through plain numbers */ - if (obj->data_in_arena) - load_attr.prog_flags |= BPF_F_ARENA_SCALAR; load_attr.fd_array = obj->fd_array; err = bpf_object__bind_arena(obj, prog, &load_attr); if (err) @@ -8991,7 +8606,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog for (i = 0; i < obj->nr_maps; i++) { map = &prog->obj->maps[i]; - if (map->libbpf_type != LIBBPF_MAP_RODATA || map->in_arena) + if (map->libbpf_type != LIBBPF_MAP_RODATA) continue; if (bpf_prog_bind_map(ret, map->fd, NULL)) { @@ -10557,7 +10172,7 @@ static void bpf_map__destroy(struct bpf_map *map) zfree(&map->init_slots); map->init_slots_sz = 0; - if (map->mmaped && map->mmaped != map->obj->arena_data && !map->in_arena) + if (map->mmaped && map->mmaped != map->obj->arena_data) munmap(map->mmaped, bpf_map_mmap_sz(map)); map->mmaped = NULL; @@ -10642,9 +10257,6 @@ void bpf_object__close(struct bpf_object *obj) close(obj->func_ptr_maps[i].fd); zfree(&obj->func_ptr_maps); zfree(&obj->func_ptrs); - for (i = 0; i < obj->data_ptr_cnt; i++) - zfree(&obj->data_ptrs[i].sym_name); - zfree(&obj->data_ptrs); if (obj->btf_module_allowlist) { for (i = 0; i < obj->btf_module_allowlist_cnt; i++) @@ -11011,8 +10623,6 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.token_fd = OPTS_GET(opts, token_fd, 0) ?: obj->token_fd; if (attr.token_fd) attr.prog_flags |= BPF_F_TOKEN_FD; - if (obj->data_in_arena) - attr.prog_flags |= BPF_F_ARENA_SCALAR; prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); if (!prog_btf_fd && obj->btf) @@ -12019,9 +11629,6 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) if (map_is_created(map)) return libbpf_err(-EBUSY); - if (map->in_arena) - return libbpf_err(-EOPNOTSUPP); - if (map->mmaped) { size_t mmap_old_sz, mmap_new_sz; int err; diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 1e4af62e8312..c99e0f623e10 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -423,16 +423,6 @@ $(LIBARENA_ASAN_SKEL): $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(LIBARENA_BPF_DEPS) +$(MAKE) -C libarena libarena_asan.skel.h $(LIBARENA_MAKE_ARGS) endif -# #![no_std] looks for compiler_builtins too. Nothing of it is used. -ifneq ($(RUST_CORE),) -$(RUST_CORE): $(RUST_CORE_SRC) - $(call msg,RUSTC,,$@) - $(Q)mkdir -p $(@D) - +$(Q)$(RUSTC_BPF) -A warnings --edition 2024 --crate-name core --out-dir $(@D) $< - +$(Q)echo '#![feature(compiler_builtins)] #![compiler_builtins] #![no_std]' | \ - $(RUSTC_BPF) -A warnings --crate-name compiler_builtins --out-dir $(@D) - -endif - # Generated test list headers define gen_tests_hdr @@ -468,7 +458,7 @@ RUNNER_PREREQS := $(INCLUDE_DIR)/vmlinux.h $(BPFOBJ) $(BPFTOOL) \ $(VERIFY_SIG_HDR) $(PRIVATE_KEY) $(VERIFICATION_CERT) \ $(LIBARENA_SKEL) $(LIBARENA_ASAN_SKEL) \ prog_tests/tests.h map_tests/tests.h \ - $(RUNNER_OBJS) $(RUST_CORE) + $(RUNNER_OBJS) # Runtime fixtures for each test_progs flavor. RUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \ diff --git a/tools/testing/selftests/bpf/Makefile.buildvars b/tools/testing/selftests/bpf/Makefile.buildvars index ff3476bc40a4..d2a0c0031b87 100644 --- a/tools/testing/selftests/bpf/Makefile.buildvars +++ b/tools/testing/selftests/bpf/Makefile.buildvars @@ -109,31 +109,6 @@ HOST_INCLUDE_DIR := $(INCLUDE_DIR) endif RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids -# Programs in Rust are built by upstream rustc. It has no prebuilt core for -# the bpf target, so it has to come with the source of core: -# rustup component add rust-src -# core is built as edition 2024, which it is since rustc 1.87. -# rustc emits LLVM bitcode and clang makes the object of it, so clang has to be -# 23 or newer and not older than LLVM of rustc. -# Otherwise RUST_CORE is empty and the tests are skipped. -RUSTC ?= rustc -RUST_CORE_SRC := $(wildcard $(shell $(RUSTC) --print sysroot 2>/dev/null)$\ - /lib/rustlib/src/rust/library/core/src/lib.rs) -ifneq ($(RUST_CORE_SRC),) -ifeq ($(shell { clang=$$(echo __clang_major__ | $(CLANG) -E -P -x c -) && \ - llvm=$$($(srctree)/scripts/rustc-llvm-version.sh $(RUSTC)) && \ - [ $$($(srctree)/scripts/rustc-version.sh $(RUSTC)) -ge 108700 ] && \ - [ $$clang -ge 23 ] && [ $$clang -ge $$((llvm / 10000)) ]; } \ - 2>/dev/null && echo y),y) -RUST_CORE := $(BUILD_DIR)/rust/libcore.rlib -endif -endif -# RUSTC_BOOTSTRAP=1 is to build core with a stable rustc, like the kernel does. -# panic=abort is a stop gap until panic=unwind is supported. -RUSTC_BPF = RUSTC_BOOTSTRAP=1 $(RUSTC) -O -C panic=abort --crate-type rlib \ - --target $(if $(IS_LITTLE_ENDIAN),bpfel,bpfeb)-unknown-none \ - -L $(dir $(RUST_CORE)) - DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool ifneq ($(CROSS_COMPILE),) CROSS_BPFTOOL := $(SCRATCH_DIR)/sbin/bpftool diff --git a/tools/testing/selftests/bpf/Makefile.skel b/tools/testing/selftests/bpf/Makefile.skel index 6570c48f92b0..e8699903feef 100644 --- a/tools/testing/selftests/bpf/Makefile.skel +++ b/tools/testing/selftests/bpf/Makefile.skel @@ -20,8 +20,7 @@ ifneq ($(BPF_CC),) BPF_SRCS := $(notdir $(wildcard progs/*.c)) BPF_OBJS := $(patsubst %.c,$(RDIR)/%.bpf.o,$(BPF_SRCS)) -SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c \ - data_in_arena_extern.c data_in_arena_nomap.c +SKEL_BLACKLIST := btf__% test_pinning_invalid.c test_sk_assign.c LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h \ linked_vars.skel.h linked_maps.skel.h linked_arena.skel.h \ @@ -138,21 +137,6 @@ $(LINKED_SKELS_H): $(RDIR)/%.skel.h: $$(addprefix $(RDIR)/,$$($$*.skel.h-deps)) $(BPFTOOL) $(GEN_SKEL) | $(RDIR) $(Q)$(cmd_bpf_link_skel) -# Programs in Rust, see Makefile.buildvars. No skeletons: the objects may be absent. -ifneq ($(RUST_CORE),) -ifeq ($(BPF_CC),$(CLANG)) -RUST_OBJS := $(patsubst progs/%.rs,$(RDIR)/%.bpf.o,$(wildcard progs/*.rs)) -BPF_OBJS += $(RUST_OBJS) - -$(RUST_OBJS): $(RDIR)/%.bpf.o: progs/%.rs $(RUST_CORE) | $(RDIR) - $(call msg,RUSTC,$(BINARY),$@) - +$(Q)$(RUSTC_BPF) --edition 2021 -C debuginfo=2 --emit=llvm-bc \ - $(patsubst -mcpu=%,-C target-cpu=%,$(filter -mcpu=%,$(BPF_CC_FLAGS))) \ - -o $(dir $(RUST_CORE))$(BINARY)-$*.bc $< && \ - $(BPF_CC) $(BPF_CC_FLAGS) -c $(dir $(RUST_CORE))$(BINARY)-$*.bc -o $@ $(call skip_on_fail,BPF) -endif -endif - # Programs in C++. No skeletons: gcc doesn't build them. ifeq ($(BPF_CC),$(CLANG)) CXX_OBJS := $(patsubst progs/%.cpp,$(RDIR)/%.bpf.o,$(wildcard progs/*.cpp)) diff --git a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c b/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c deleted file mode 100644 index 2ac2e9a652fe..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/arena_scalar_blinded.c +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include "sysctl_helpers.h" -#include "verifier_arena_scalar.skel.h" - -/* The same tests with constants of the programs blinded */ -void serial_test_arena_scalar_blinded(void) -{ - const char *harden = "/proc/sys/net/core/bpf_jit_harden"; - char old[16] = {}; - - if (!is_jit_enabled()) { - test__skip(); - return; - } - if (sysctl_set_or_fail(harden, old, "2")) - return; - RUN_TESTS(verifier_arena_scalar); - sysctl_set_or_fail(harden, NULL, old); -} diff --git a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c b/tools/testing/selftests/bpf/prog_tests/data_in_arena.c deleted file mode 100644 index cb1023507c01..000000000000 --- a/tools/testing/selftests/bpf/prog_tests/data_in_arena.c +++ /dev/null @@ -1,216 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include "data_in_arena.skel.h" -#include "data_in_arena_decl.skel.h" -#include "data_in_arena_fail.skel.h" - -static int run_prog(struct bpf_program *prog) -{ - LIBBPF_OPTS(bpf_test_run_opts, topts); - - if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &topts), "test_run")) - return -1; - return topts.retval; -} - -static void run(struct data_in_arena *skel, int counter) -{ - int i; - - ASSERT_EQ(run_prog(skel->progs.use_data), counter + 7 + 1 + 2, "retval"); - ASSERT_EQ(skel->bss->sum, counter + 7 + 1 + 2, "sum"); - ASSERT_EQ(skel->data->counter, counter + 1, "counter"); - ASSERT_EQ(skel->data->pair[1], 5, "pair[1]"); - for (i = 0; i < 4; i++) - ASSERT_EQ(skel->bss->table[i], 10 * (i + 1) + i, "table"); -} - -static void test_in_arena(void) -{ - struct bpf_map_info info = {}; - struct data_in_arena *skel; - __u32 len = sizeof(info); - struct bpf_map *arena; - size_t sz; - - skel = data_in_arena__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - - arena = bpf_object__find_map_by_name(skel->obj, "arena"); - if (!ASSERT_OK_PTR(arena, "arena")) - goto out; - ASSERT_EQ(bpf_map__type(arena), BPF_MAP_TYPE_ARENA, "arena type"); - ASSERT_EQ(bpf_map__max_entries(arena), 1, "arena pages"); - ASSERT_OK(bpf_map__set_max_entries(arena, 8), "arena resize"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.data), "data autocreate"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.bss), "bss autocreate"); - ASSERT_FALSE(bpf_map__autocreate(skel->maps.rodata), "rodata autocreate"); - ASSERT_EQ(bpf_map__set_autocreate(skel->maps.data, true), -EOPNOTSUPP, "set_autocreate"); - ASSERT_EQ(bpf_map__set_value_size(skel->maps.bss, 4096), -EOPNOTSUPP, "set_value_size"); - ASSERT_EQ(bpf_map__initial_value(skel->maps.data, &sz), skel->data, "initial_value"); - ASSERT_EQ(sz, sizeof(*skel->data), "initial_value size"); - - /* initial values are set the usual way */ - skel->data->counter = 100; - - if (!ASSERT_OK(data_in_arena__load(skel), "load")) - goto out; - /* there are no maps behind the sections */ - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.data), &info, &len), "data map"); - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.bss), &info, &len), "bss map"); - ASSERT_ERR(bpf_map_get_info_by_fd(bpf_map__fd(skel->maps.rodata), &info, &len), - "rodata map"); - run(skel, 100); - - /* pointers to data next to pointers to functions */ - ASSERT_EQ(run_prog(skel->progs.use_ops), 42 + 1 + 'e', "use_ops"); - ASSERT_EQ(skel->data->counter, 102, "counter"); - - /* alignment of sections and pointers to data in data */ - ASSERT_EQ((unsigned long)&skel->bss->aligned64 % 64, 0, "alignment"); - ASSERT_EQ(run_prog(skel->progs.use_ptrs), 0, "use_ptrs"); - ASSERT_EQ(skel->data->x, 43, "x"); - ASSERT_EQ(skel->bss->aligned64.v[7], 7, "aligned64"); - ASSERT_EQ(skel->data->px, &skel->data->x, "px"); - - /* format strings of bpf_printk() and BPF_SNPRINTF() */ - ASSERT_EQ(run_prog(skel->progs.use_printk), sizeof("43-7"), "use_printk"); - ASSERT_STREQ(skel->bss->out, "43-7", "out"); -out: - data_in_arena__destroy(skel); -} - -/* The object has an arena map and __arena variables */ -static void test_declared_arena(void) -{ - struct data_in_arena_decl *skel; - struct bpf_map *map; - int arenas = 0; - - skel = data_in_arena_decl__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - bpf_object__for_each_map(map, skel->obj) - arenas += bpf_map__type(map) == BPF_MAP_TYPE_ARENA; - ASSERT_EQ(arenas, 1, "no second arena"); - skel->data->counter = 6; - if (!ASSERT_OK(data_in_arena_decl__load(skel), "load")) - goto out; - ASSERT_EQ(run_prog(skel->progs.use_data), 6 + 7 + 11, "retval"); - ASSERT_EQ(run_prog(skel->progs.use_data), 7 + 7 + 12, "retval"); - ASSERT_EQ(skel->bss->sum, 7 + 7 + 12, "sum"); - ASSERT_EQ(skel->data->counter, 8, "counter"); -out: - data_in_arena_decl__destroy(skel); -} - -/* A pointer in data that can't be made an address of arena fails the load */ -static void test_ptr_to_map(void) -{ - struct data_in_arena_fail *skel; - - skel = data_in_arena_fail__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - ASSERT_ERR(data_in_arena_fail__load(skel), "load"); - data_in_arena_fail__destroy(skel); -} - -/* So does a pointer to a variable of the kernel. There is no skeleton: the open fails. */ -static void test_ptr_to_extern(void) -{ - struct bpf_object *obj; - - obj = bpf_object__open_file("./data_in_arena_extern.bpf.o", NULL); - if (!ASSERT_ERR_PTR(obj, "open")) - bpf_object__close(obj); -} - -/* __arena variables and no arena map. There is no skeleton: the open fails. */ -static void test_arena_var_no_map(void) -{ - struct bpf_object *obj; - - obj = bpf_object__open_file("./data_in_arena_nomap.bpf.o", NULL); - if (!ASSERT_ERR_PTR(obj, "open")) - bpf_object__close(obj); -} - -/* The address of an arena that libbpf doesn't create is not known. No pointers in data then. */ -static void test_not_my_arena(bool pin) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); - struct data_in_arena *skel; - struct bpf_map *arena; - int fd = -1; - - skel = data_in_arena__open(); - if (!ASSERT_OK_PTR(skel, "open")) - return; - arena = bpf_object__find_map_by_name(skel->obj, "arena"); - if (!ASSERT_OK_PTR(arena, "arena")) - goto out; - if (pin) { - ASSERT_OK(bpf_map__set_pin_path(arena, "/sys/fs/bpf/data_in_arena"), "pin_path"); - } else { - fd = bpf_map_create(BPF_MAP_TYPE_ARENA, "arena", 0, 0, 1, &opts); - if (!ASSERT_GE(fd, 0, "map_create")) - goto out; - ASSERT_OK(bpf_map__reuse_fd(arena, fd), "reuse_fd"); - } - ASSERT_EQ(data_in_arena__load(skel), -ENOTSUP, "load"); -out: - if (fd >= 0) - close(fd); - data_in_arena__destroy(skel); -} - -/* The object is there if rustc and clang can build it, see Makefile.buildvars */ -static void test_rust(void) -{ - const char *file = "./data_in_arena_rust.bpf.o"; - struct bpf_object *obj; - - if (access(file, R_OK)) { - test__skip(); - return; - } - obj = bpf_object__open_file(file, NULL); - if (!ASSERT_OK_PTR(obj, "open")) - return; - if (!ASSERT_OK(bpf_object__load(obj), "load")) - goto out; - /* libbpf goes on without BTF when the kernel doesn't take it */ - ASSERT_GE(bpf_object__btf_fd(obj), 0, "btf_fd"); - ASSERT_EQ(run_prog(bpf_object__find_program_by_name(obj, "list_in_data")), - 100 + 20 + 3, "retval"); -out: - bpf_object__close(obj); -} - -void test_data_in_arena(void) -{ -#if !defined(__x86_64__) && !defined(__aarch64__) - /* other JITs don't take BPF_F_ARENA_SCALAR */ - test__skip(); - return; -#endif - if (test__start_subtest("arena")) - test_in_arena(); - if (test__start_subtest("declared_arena")) - test_declared_arena(); - if (test__start_subtest("ptr_to_map")) - test_ptr_to_map(); - if (test__start_subtest("ptr_to_extern")) - test_ptr_to_extern(); - if (test__start_subtest("arena_var_no_map")) - test_arena_var_no_map(); - if (test__start_subtest("pinned_arena")) - test_not_my_arena(true); - if (test__start_subtest("reused_arena")) - test_not_my_arena(false); - if (test__start_subtest("rust")) - test_rust(); -} diff --git a/tools/testing/selftests/bpf/prog_tests/jit_insns.c b/tools/testing/selftests/bpf/prog_tests/jit_insns.c index dabefdb8a353..52bffe27199b 100644 --- a/tools/testing/selftests/bpf/prog_tests/jit_insns.c +++ b/tools/testing/selftests/bpf/prog_tests/jit_insns.c @@ -2881,6 +2881,63 @@ static void test_convergence(void) } } +/* + * On x86-64 a backward jump sized from its own previous-pass address grew + * here while others shrank, so two passes gave the same image length with + * jumps still changing. The fillers are 7 and 2 bytes of native code, the + * conditional jumps are never taken. + */ +static void test_same_len_not_converged(void) +{ + struct bpf_insn insns[] = { + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, IN0), + BPF_LDX_MEM(BPF_DW, BPF_REG_4, BPF_REG_1, IN1), + BPF_JMP_IMM(BPF_JA, 0, 0, 1), + BPF_JMP_IMM(BPF_JA, 0, 0, 29), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 20), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, -3), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 22), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 2), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 23), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP_IMM(BPF_JA, 0, 0, -22), + BPF_MOV64_IMM(BPF_REG_2, -1), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 3), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_JMP32_REG(BPF_JEQ, BPF_REG_0, BPF_REG_4, 2), + BPF_JMP_IMM(BPF_JA, 0, 0, -22), + BPF_MOV32_IMM(BPF_REG_2, 0), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 10), + BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, RES), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int fd; + + fd = load_prog(insns, ARRAY_SIZE(insns), "same len not converged"); + if (fd < 0) + return; + ASSERT_GT(jited_len(fd), 0, "same len not converged jited"); + prog_run(fd, 1, 2, 11, "same len not converged"); + close(fd); +} + /* * Calls into a subprogram with all five arguments, which clobbers a subset * of r6-r9 and uses the deepest stack slot the budget leaves, directly or @@ -3087,6 +3144,8 @@ void test_jit_insns(void) test_max_jmp(); if (test__start_subtest("convergence")) test_convergence(); + if (test__start_subtest("same_len_not_converged")) + test_same_len_not_converged(); if (test__start_subtest("calls")) test_calls(); if (test__start_subtest("helper_calls")) diff --git a/tools/testing/selftests/bpf/prog_tests/map_kptr.c b/tools/testing/selftests/bpf/prog_tests/map_kptr.c index 17e707dddda8..f103f8cd458b 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_kptr.c +++ b/tools/testing/selftests/bpf/prog_tests/map_kptr.c @@ -225,6 +225,11 @@ void serial_test_map_kptr(void) /* Observe refcount dropping to 1 on map release. */ test_map_kptr_success(true); + + /* The next test counts the same references */ + ASSERT_OK(kern_sync_rcu_tasks_trace(skel), "sync rcu_tasks_trace"); + ASSERT_OK(kern_sync_rcu(), "sync rcu"); + wait_for_map_release(); } rcu_tasks_trace_gp__destroy(skel); diff --git a/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c b/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c index de23a14b3d40..10b4ea913921 100644 --- a/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c +++ b/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c @@ -23,10 +23,17 @@ static bool have_tree_srcu(void) return kallsyms_find("srcu_expedite_current", &addr) == 0; } +/* Arm the handler for this thread, then trigger call_rcu_tasks_trace(). */ +static noinline int arm_and_delete(struct rcu_reentry *skel, int pid, int map_fd, int *pidfd) +{ + skel->bss->target_pid = pid; + return bpf_map_delete_elem(map_fd, pidfd); +} + void test_rcu_reentry(void) { struct rcu_reentry *skel; - int err, pidfd = -1, map_fd; + int err, pidfd = -1, bad_fd = -1, map_fd; cpu_set_t set, old_set; bool affinity_saved; __u64 val = 1; @@ -63,9 +70,13 @@ void test_rcu_reentry(void) if (!ASSERT_OK(err, "boot_create")) goto restore; - /* Arm the handler for this thread, then trigger call_rcu_tasks_trace(). */ - skel->bss->target_pid = syscall(__NR_gettid); - err = bpf_map_delete_elem(map_fd, &pidfd); + /* + * The handler fires at the first rcu_segcblist_enqueue() in this thread, and it deletes + * the element. A page fault can get there too, when the file system reads the page. + * Go through the same code first, without a pid, to bring its text and .bss in. + */ + arm_and_delete(skel, 0, map_fd, &bad_fd); + err = arm_and_delete(skel, syscall(__NR_gettid), map_fd, &pidfd); if (!ASSERT_OK(err, "boot_delete")) goto restore; diff --git a/tools/testing/selftests/bpf/prog_tests/send_signal.c b/tools/testing/selftests/bpf/prog_tests/send_signal.c index 7ac4d5a488aa..ecec2ee8b19a 100644 --- a/tools/testing/selftests/bpf/prog_tests/send_signal.c +++ b/tools/testing/selftests/bpf/prog_tests/send_signal.c @@ -261,6 +261,110 @@ static void test_send_signal_nmi(bool signal_thread, bool remote) test_send_signal_common(&attr, signal_thread, remote); } +static void test_send_signal_kworker(void) +{ + struct test_send_signal_kern *skel; + int pipe_c2p[2], pipe_p2c[2]; + struct sigaction sa = {}; + char buf[256]; + int err = -1; + pid_t pid; + + if (!ASSERT_OK(pipe(pipe_c2p), "pipe_c2p")) + return; + + if (!ASSERT_OK(pipe(pipe_p2c), "pipe_p2c")) { + close(pipe_c2p[0]); + close(pipe_c2p[1]); + return; + } + + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) { + close(pipe_c2p[0]); + close(pipe_c2p[1]); + close(pipe_p2c[0]); + close(pipe_p2c[1]); + return; + } + + if (pid == 0) { + /* install signal handler and notify parent */ + sa.sa_sigaction = sigusr1_siginfo_handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + ASSERT_NEQ(sigaction(SIGUSR1, &sa, NULL), -1, "sigaction"); + + close(pipe_c2p[0]); /* close read */ + close(pipe_p2c[1]); /* close write */ + + /* notify parent signal handler is installed */ + ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write"); + + /* make sure parent enabled bpf program to send_signal */ + ASSERT_EQ(read(pipe_p2c[0], buf, 1), 1, "pipe_read"); + + /* + * The signal is sent from workqueue context, so nothing has + * to be triggered from here. + */ + while (!sigusr1_received) + sleep(1); + + buf[0] = sigusr1_received; + + ASSERT_EQ(sigusr1_received, 8, "sigusr1_received"); + ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write"); + + close(pipe_c2p[1]); + close(pipe_p2c[0]); + exit(0); + } + + close(pipe_c2p[1]); /* close write */ + close(pipe_p2c[0]); /* close read */ + + skel = test_send_signal_kern__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + goto skel_open_load_failure; + + /* wait until child signal handler installed */ + ASSERT_EQ(read(pipe_c2p[0], buf, 1), 1, "pipe_read"); + + /* pid == 0 keeps the other programs of this skeleton inactive */ + skel->bss->kworker_test = 1; + skel->bss->pid = 0; + skel->bss->sig = SIGUSR1; + skel->bss->target_pid = pid; + + err = test_send_signal_kern__attach(skel); + if (!ASSERT_OK(err, "skel_attach")) { + err = -1; + goto destroy_skel; + } + + /* notify child that bpf program can send_signal now */ + ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write"); + + /* wait for result, workqueues run on their own */ + err = read_with_timeout(pipe_c2p[0], buf, 1, 10 * 1000 * 1000); + if (!ASSERT_GT(err, 0, "reading pipe")) + goto destroy_skel; + + ASSERT_EQ(buf[0], 8, "incorrect result"); + +destroy_skel: + test_send_signal_kern__destroy(skel); +skel_open_load_failure: + close(pipe_c2p[0]); + close(pipe_p2c[1]); + /* + * Child is either about to exit cleanly or stuck in case of errors. + * Nudge it to exit. + */ + kill(pid, SIGKILL); + wait(NULL); +} + void test_send_signal(void) { if (test__start_subtest("send_signal_tracepoint")) @@ -289,4 +393,6 @@ void test_send_signal(void) test_send_signal_perf(true, true); if (test__start_subtest("send_signal_nmi_thread_remote")) test_send_signal_nmi(true, true); + if (test__start_subtest("send_signal_kworker")) + test_send_signal_kworker(); } diff --git a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c index 461ded722351..9b646d134a45 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c +++ b/tools/testing/selftests/bpf/prog_tests/test_struct_ops_assoc.c @@ -136,7 +136,8 @@ static void test_st_ops_assoc_in_timer(void) struct_ops_assoc_in_timer__destroy(skel); } -static void test_st_ops_assoc_in_timer_no_uref(void) +static void run_st_ops_assoc_in_timer_no_uref(int timer_ns, int wait_map_free, + int spin_loops) { struct struct_ops_assoc_in_timer *skel = NULL; struct bpf_link *link; @@ -148,7 +149,8 @@ static void test_st_ops_assoc_in_timer_no_uref(void) err = bpf_program__assoc_struct_ops(skel->progs.syscall_prog, skel->maps.st_ops_map, NULL); - ASSERT_OK(err, "bpf_program__assoc_struct_ops"); + if (!ASSERT_OK(err, "bpf_program__assoc_struct_ops")) + goto out; link = bpf_map__attach_struct_ops(skel->maps.st_ops_map); if (!ASSERT_OK_PTR(link, "bpf_map__attach_struct_ops")) @@ -157,12 +159,18 @@ static void test_st_ops_assoc_in_timer_no_uref(void) /* * Run .test_1 by calling kfunc bpf_kfunc_multi_st_ops_test_1_prog_arg() and checks * the return value. .test_1 will also schedule timer_cb that runs .test_1 again. - * timer_cb will run 500ms after syscall_prog runs, when the user space no longer - * holds a reference to st_ops_map. + * timer_cb will run timer_ns after syscall_prog runs and, with wait_map_free, every + * timer_ns after that until the map is freed. With 500ms, it first runs when the + * user space no longer holds a reference to st_ops_map. */ - skel->bss->timer_ns = 500000000; + skel->bss->timer_ns = timer_ns; + skel->bss->wait_map_free = wait_map_free; + skel->bss->spin_loops = spin_loops; err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.syscall_prog), NULL); - ASSERT_OK(err, "bpf_prog_test_run_opts"); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) { + bpf_link__destroy(link); + goto out; + } /* Detach and close struct_ops map to cause it to be freed */ bpf_link__destroy(link); @@ -178,6 +186,21 @@ static void test_st_ops_assoc_in_timer_no_uref(void) struct_ops_assoc_in_timer__destroy(skel); } +static void test_st_ops_assoc_in_timer_no_uref(void) +{ + run_st_ops_assoc_in_timer_no_uref(500000000, 1, 0); +} + +/* + * Keep calling .test_1 through the association from the timer callback, and + * stay in it for a while each time, while the map is being freed. The map + * must not be freed under a running .test_1. + */ +static void test_st_ops_assoc_in_timer_free(void) +{ + run_st_ops_assoc_in_timer_no_uref(0, 1, 1 << 20); +} + void test_struct_ops_assoc(void) { if (test__start_subtest("st_ops_assoc")) @@ -188,4 +211,6 @@ void test_struct_ops_assoc(void) test_st_ops_assoc_in_timer(); if (test__start_subtest("st_ops_assoc_in_timer_no_uref")) test_st_ops_assoc_in_timer_no_uref(); + if (test__start_subtest("st_ops_assoc_in_timer_free")) + test_st_ops_assoc_in_timer_free(); } diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 04740ba37e91..6a494c381266 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -13,7 +13,6 @@ #include "verifier_and.skel.h" #include "verifier_arena.skel.h" #include "verifier_arena_large.skel.h" -#include "verifier_arena_scalar.skel.h" #include "verifier_arena_globals1.skel.h" #include "verifier_arena_globals2.skel.h" #include "verifier_array_access.skel.h" @@ -207,7 +206,6 @@ void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } void test_verifier_arena_large(void) { RUN(verifier_arena_large); } -void test_verifier_arena_scalar(void) { RUN(verifier_arena_scalar); } void test_verifier_arena_globals1(void) { RUN(verifier_arena_globals1); } void test_verifier_arena_globals2(void) { RUN(verifier_arena_globals2); } void test_verifier_basic_stack(void) { RUN(verifier_basic_stack); } diff --git a/tools/testing/selftests/bpf/progs/arena_noarena.c b/tools/testing/selftests/bpf/progs/arena_noarena.c index edbd40cd5be8..1f46e5aa403b 100644 --- a/tools/testing/selftests/bpf/progs/arena_noarena.c +++ b/tools/testing/selftests/bpf/progs/arena_noarena.c @@ -21,19 +21,4 @@ int arena_arg_no_arena(void *ctx) return 0; } -SEC("syscall") -__flag(BPF_F_ARENA_SCALAR) -__arch_x86_64 -__arch_arm64 -__description("arena_scalar: no arena, no access through a number") -__failure __msg("R6 invalid mem access 'scalar'") -__naked void no_arena(void) -{ - asm volatile (" \ - r6 = 0x100000000000 ll; \ - r0 = *(u64 *)(r6 + 0); \ - exit; \ -" ::: __clobber_all); -} - char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c index a980a3a399d3..f4fb8f81afa7 100644 --- a/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c @@ -26,6 +26,11 @@ void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, int g) { } +int callback_6args(long a, long b, long c, long d, long e, long f) +{ + return a + b + c + d + e + f; +} + #else int subprog_bad_order_6args(void) @@ -47,4 +52,9 @@ void subprog_bad_ptr_7args(void) { } +int callback_6args(void) +{ + return 0; +} + #endif diff --git a/tools/testing/selftests/bpf/progs/data_in_arena.c b/tools/testing/selftests/bpf/progs/data_in_arena.c deleted file mode 100644 index 18f38103c0fc..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena.c +++ /dev/null @@ -1,112 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -long pair[2] = { 1, 2 }; -int x = 42; -long sum; -long table[4]; -struct { - long v[8]; -} aligned64 __attribute__((aligned(64))); -const volatile int ro = 7; -const volatile long ro_table[4] = { 10, 20, 30, 40 }; - -/* const strings stay in a map for helpers and kfuncs, a copy of them is in arena */ -const char hello[] SEC(".rodata.str.hello") = "hello"; - -/* pointers to data that are stored in data */ -int *px = &x; -const char *str = hello; -int *const volatile cpx SEC(".data.rel.ro") = &x; - -SEC("syscall") -int use_data(void *ctx) -{ - int i; - - for (i = 0; i < 4; i++) - table[i] = ro_table[i] + i; - sum = counter + ro + pair[0] + pair[1]; - counter++; - __sync_fetch_and_add(&pair[1], 3); - return sum; -} - -typedef int (*op_fn)(int); - -static __noinline int add1(int v) -{ - return v + 1; -} - -/* - * Pointers to functions and to data in read-only data of a program with callx. - * Volatile, so that the compiler doesn't replace the pointers with what - * they point to. - */ -static const volatile struct { - op_fn fn; - int *data; - const char *name; -} ops SEC(".data.rel.ro") = { add1, &x, hello }; - -SEC("syscall") -int use_ops(void *ctx) -{ - /* a program has the arena when its code refers to it */ - counter++; -#ifdef __clang__ - return ops.fn(*ops.data) + ops.name[1]; -#else - /* gcc doesn't support indirect calls */ - return add1(*ops.data) + ops.name[1]; -#endif -} - -SEC("syscall") -int use_ptrs(void *ctx) -{ - unsigned long addr = (unsigned long)&aligned64; - char local[4] = "abc"; - - /* hide the address from the compiler, it knows that '& 63' is 0 */ - asm volatile ("" : "+r"(addr)); - if (addr & 63) - return 1; - aligned64.v[7] = 7; - if (*px != 42) - return 2; - *px = 43; - if (x != 43 || *cpx != 43) - return 3; - if (str[0] != 'h' || str[4] != 'o' || str[5]) - return 4; - /* the literal is in a map */ - if (bpf_strncmp(local, sizeof(local), "abc")) - return 5; - return 0; -} - -char out[16]; - -/* format strings are in a map */ -SEC("syscall") -int use_printk(void *ctx) -{ - char buf[sizeof(out)]; - int i, n; - - bpf_printk("counter %d", counter); - n = BPF_SNPRINTF(buf, sizeof(buf), "%d-%d", x, ro); - for (i = 0; i < sizeof(out); i++) - out[i] = buf[i]; - return n; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c b/tools/testing/selftests/bpf/progs/data_in_arena_decl.c deleted file mode 100644 index 01bc4fea8f0c..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_decl.c +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#define BPF_NO_KFUNC_PROTOTYPES -#include -#include -#include "bpf_experimental.h" -#include - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 4); -} arena SEC(".maps"); - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -long sum; -const volatile int ro = 7; - -#if defined(__BPF_FEATURE_ADDR_SPACE_CAST) -int __arena avar = 11; -#else -int avar = 11; -#endif - -SEC("syscall") -int use_data(void *ctx) -{ - sum = counter + ro + avar; - counter++; - avar++; - return sum; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c b/tools/testing/selftests/bpf/progs/data_in_arena_extern.c deleted file mode 100644 index dc1ad5ca3bdd..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_extern.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -extern const int bpf_prog_active __ksym; - -/* the variable of the kernel is not in arena */ -const void *kp = &bpf_prog_active; - -SEC("syscall") -int ptr_to_extern(void *ctx) -{ - return *(int *)kp; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c b/tools/testing/selftests/bpf/progs/data_in_arena_fail.c deleted file mode 100644 index ad8afe9afc96..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_fail.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int x = 42; -/* the table is read-only data with pointers. It's not in arena. */ -int *const volatile tbl[1] SEC(".data.rel.ro") = { &x }; -int *const volatile *pp = tbl; - -SEC("syscall") -int ptr_to_map(void *ctx) -{ - return **pp; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c b/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c deleted file mode 100644 index ccf66321eb36..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_nomap.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "bpf_arena_common.h" - -/* global data of the object is in arena */ -char data_in_arena SEC(".arena.data"); - -int counter = 5; -/* needs an arena map that is declared. The one that libbpf creates won't do. */ -int __arena_global avar = 11; - -SEC("syscall") -int arena_var(void *ctx) -{ - return avar + counter; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs b/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs deleted file mode 100644 index 3dd9d4208232..000000000000 --- a/tools/testing/selftests/bpf/progs/data_in_arena_rust.rs +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -// Why .data, .bss and .rodata of a program in Rust are in arena. -// -// A reference in Rust is an address. It doesn't say what it points to and it -// can be stored in data. The list below has a node in each of the sections. -// The nodes are linked by references that are in the data: -// IN_BSS.next is stored by the program, -// IN_DATA.next is a relocation in .data against .rodata. -// sum() loads the references back and reads the three nodes with the same insn. -// -// When the sections are array maps libbpf skips the relocation in .data, and -// what sum() loads from a node is a number that can't be dereferenced: -// R1 invalid mem access 'scalar' -// In arena the address of a node is a number to begin with. - -#![no_std] -#![no_main] - -// Tell libbpf to keep .data, .bss and .rodata in arena. -#[used] -#[link_section = ".arena.data"] -static DATA_IN_ARENA: u8 = 0; - -#[used] -#[link_section = "license"] -static LICENSE: [u8; 4] = *b"GPL\0"; - -// panic=abort is a stop gap until panic=unwind is supported. -// Nothing here panics, so the handler is not a part of the program. -#[panic_handler] -fn panic(_info: &core::panic::PanicInfo) -> ! { - loop {} -} - -pub struct Node { - val: u32, - next: Option<&'static Node>, -} - -// no_mangle makes them visible outside, so LLVM can't fold the list into a constant. -#[no_mangle] -static IN_RODATA: Node = Node { val: 3, next: None }; -#[no_mangle] -static mut IN_DATA: Node = Node { - val: 20, - next: Some(&IN_RODATA), -}; -#[no_mangle] -static mut IN_BSS: Node = Node { val: 0, next: None }; - -#[inline(never)] -fn sum(mut node: Option<&Node>) -> u32 { - let mut sum = 0; - // The verifier wants a bound. - for _ in 0..8 { - let Some(n) = node else { break }; - sum += n.val; - node = n.next; - } - sum -} - -#[no_mangle] -#[link_section = "syscall"] -pub extern "C" fn list_in_data(_ctx: *mut u8) -> u32 { - unsafe { - let head = &mut *&raw mut IN_BSS; - head.val = 100; - head.next = Some(&*&raw const IN_DATA); - sum(Some(head)) - } -} diff --git a/tools/testing/selftests/bpf/progs/iters_testmod.c b/tools/testing/selftests/bpf/progs/iters_testmod.c index 145750bb64ef..8896335668b4 100644 --- a/tools/testing/selftests/bpf/progs/iters_testmod.c +++ b/tools/testing/selftests/bpf/progs/iters_testmod.c @@ -8,6 +8,44 @@ char _license[] SEC("license") = "GPL"; +struct bpf_iter_testmod_current { + u64 __opaque; +}; + +SEC("raw_tp/sys_enter") +__success +int iter_independent_lifetime_after_next(const void *ctx) +{ + struct bpf_iter_testmod_current it; + struct task_struct *task; + + bpf_iter_testmod_current_new(&it); + task = bpf_iter_testmod_current_next(&it); + if (!task) + goto out; + + bpf_iter_testmod_current_next(&it); + bpf_kfunc_trusted_task_test(task); +out: + bpf_iter_testmod_current_destroy(&it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_independent_lifetime_after_destroy(const void *ctx) +{ + struct bpf_iter_testmod_current it; + struct task_struct *task; + + bpf_iter_testmod_current_new(&it); + task = bpf_iter_testmod_current_next(&it); + bpf_iter_testmod_current_destroy(&it); + if (task) + bpf_kfunc_trusted_task_test(task); + return 0; +} + SEC("raw_tp/sys_enter") __success int iter_next_trusted(const void *ctx) @@ -28,6 +66,298 @@ int iter_next_trusted(const void *ctx) return 0; } +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + bpf_iter_task_vma_destroy(&vma_it); + return vma_ptr->vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + u64 vm_start; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!next_vma_ptr) + goto out; + + vm_start = vma_ptr->vm_start; + bpf_iter_task_vma_destroy(&vma_it); + return vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_after_next_null(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + u64 vm_start; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (next_vma_ptr) + goto out; + + vm_start = vma_ptr->vm_start; + bpf_iter_task_vma_destroy(&vma_it); + return vm_start; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_field_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + bpf_iter_task_vma_destroy(&vma_it); + if (file_ptr) + return file_ptr->f_mode; + return 0; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("invalid mem access 'scalar'") +int iter_next_trusted_field_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr, *next_vma_ptr; + struct file *file_ptr; + u32 mode; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + next_vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!next_vma_ptr) + goto out; + + mode = file_ptr->f_mode; + bpf_iter_task_vma_destroy(&vma_it); + return mode; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("Leaking reference id=") +int iter_next_file_dynptr_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_iter_task_vma_next(&vma_it); + bpf_dynptr_file_discard(&dynptr); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__failure __msg("Leaking reference id=") +int iter_next_file_dynptr_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_iter_task_vma_destroy(&vma_it); + bpf_dynptr_file_discard(&dynptr); + return 0; +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_file_dynptr_discard_before_advance(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct bpf_dynptr dynptr; + struct file *file_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + file_ptr = vma_ptr->vm_file; + if (!file_ptr) + goto out; + + bpf_dynptr_from_file(file_ptr, 0, &dynptr); + bpf_dynptr_file_discard(&dynptr); + bpf_iter_task_vma_next(&vma_it); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_current_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (!vma_ptr) + goto out; + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) + bpf_kfunc_trusted_vma_test(vma_ptr); +out: + bpf_iter_task_vma_destroy(&vma_it); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_rcu_field_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct mm_struct *mm_ptr; + struct file *file_ptr = NULL; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) { + mm_ptr = vma_ptr->vm_mm; + /* exe_file has RCU protection independent of the iterator. */ + if (mm_ptr) + file_ptr = mm_ptr->exe_file; + } + + bpf_iter_task_vma_destroy(&vma_it); + if (file_ptr) + return file_ptr->f_mode; + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_trusted_rcu_field_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task_vma vma_it; + struct vm_area_struct *vma_ptr; + struct mm_struct *mm_ptr; + struct file *file_ptr = NULL; + u32 mode = 0; + + bpf_iter_task_vma_new(&vma_it, cur_task, 0); + + vma_ptr = bpf_iter_task_vma_next(&vma_it); + if (vma_ptr) { + mm_ptr = vma_ptr->vm_mm; + if (mm_ptr) + file_ptr = mm_ptr->exe_file; + } + + bpf_iter_task_vma_next(&vma_it); + if (file_ptr) + mode = file_ptr->f_mode; + bpf_iter_task_vma_destroy(&vma_it); + return mode; +} + SEC("raw_tp/sys_enter") __failure __msg("Possibly NULL pointer passed to trusted R1") int iter_next_trusted_or_null(const void *ctx) @@ -77,6 +407,46 @@ int iter_next_rcu(const void *ctx) return 0; } +SEC("raw_tp/sys_enter") +__success +int iter_next_rcu_after_destroy(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task task_it; + struct task_struct *task_ptr; + + bpf_iter_task_new(&task_it, cur_task, 0); + + task_ptr = bpf_iter_task_next(&task_it); + bpf_iter_task_destroy(&task_it); + if (!task_ptr) + return 0; + + bpf_kfunc_rcu_task_test(task_ptr); + return 0; +} + +SEC("raw_tp/sys_enter") +__success +int iter_next_rcu_after_next(const void *ctx) +{ + struct task_struct *cur_task = bpf_get_current_task_btf(); + struct bpf_iter_task task_it; + struct task_struct *task_ptr; + + bpf_iter_task_new(&task_it, cur_task, 0); + + task_ptr = bpf_iter_task_next(&task_it); + if (!task_ptr) + goto out; + + bpf_iter_task_next(&task_it); + bpf_kfunc_rcu_task_test(task_ptr); +out: + bpf_iter_task_destroy(&task_it); + return 0; +} + SEC("raw_tp/sys_enter") __failure __msg("Possibly NULL pointer passed to trusted R1") int iter_next_rcu_or_null(const void *ctx) diff --git a/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c b/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c index 0bed49e9f217..47a44c8e8b11 100644 --- a/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c +++ b/tools/testing/selftests/bpf/progs/struct_ops_assoc_in_timer.c @@ -25,6 +25,13 @@ int test_err; int timer_ns; int timer_test_1_ret; int timer_cb_run; +int wait_map_free; +int spin_loops; + +static int spin(u32 i, void *ctx) +{ + return 0; +} __noinline static int timer_cb(void *map, int *key, struct bpf_timer *timer) { @@ -34,6 +41,12 @@ __noinline static int timer_cb(void *map, int *key, struct bpf_timer *timer) timer_test_1_ret = bpf_kfunc_multi_st_ops_test_1_assoc(&args); recur--; + /* Try again later until the map is freed */ + if (wait_map_free && timer_test_1_ret != -1) { + bpf_timer_start(timer, timer_ns, 0); + return 0; + } + timer_cb_run++; return 0; @@ -53,6 +66,9 @@ int BPF_PROG(test_1, struct st_ops_args *args) bpf_timer_init(timer, &array_map, 1); bpf_timer_set_callback(timer, timer_cb); bpf_timer_start(timer, timer_ns, 0); + } else if (spin_loops) { + /* Stay in the trampoline while the map may be freed */ + bpf_loop(spin_loops, spin, NULL, 0); } return MAP_MAGIC; diff --git a/tools/testing/selftests/bpf/progs/test_send_signal_kern.c b/tools/testing/selftests/bpf/progs/test_send_signal_kern.c index 176a355e3062..f9c4496d9fe1 100644 --- a/tools/testing/selftests/bpf/progs/test_send_signal_kern.c +++ b/tools/testing/selftests/bpf/progs/test_send_signal_kern.c @@ -9,6 +9,7 @@ void bpf_task_release(struct task_struct *p) __ksym; int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type, u64 value) __ksym; __u32 sig = 0, pid = 0, status = 0, signal_thread = 0, target_pid = 0; +__u32 kworker_test = 0; static __always_inline int bpf_send_signal_test(void *ctx) { @@ -60,6 +61,34 @@ int send_signal_tp_sched(void *ctx) return bpf_send_signal_test(ctx); } +/* + * Send a signal to a task other than current, from a context where current + * is a kernel thread. No filtering on current is possible here, so this is + * driven by target_pid, and gated on kworker_test so that it stays inactive + * in the other tests. + */ +SEC("tp_btf/workqueue_execute_start") +int send_signal_kworker(void *ctx) +{ + struct task_struct *target_task; + int ret; + + if (status != 0 || !kworker_test || target_pid == 0) + return 0; + + target_task = bpf_task_from_pid(target_pid); + if (!target_task) + return 0; + + ret = bpf_send_signal_task(target_task, sig, PIDTYPE_TGID, 8); + bpf_task_release(target_task); + + if (ret == 0) + status = 1; + + return 0; +} + SEC("perf_event") int send_signal_perf(void *ctx) { diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 1ce9c4b78481..b17a6dfc100d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -610,6 +610,21 @@ int mixed_arena_scalar_alu64_arena_first(void *ctx) return 0; } +SEC("syscall") +__description("arena: a number is not an address in arena") +__failure __msg("R6 invalid mem access 'scalar'") +__naked void access_through_number(void) +{ + asm volatile (" \ + r1 = %[arena] ll; \ + r6 = 0x100000000000 ll; \ + r0 = *(u64 *)(r6 + 0); \ + exit; \ +" : + : __imm_addr(arena) + : __clobber_all); +} + SEC("syscall") __success __retval(0) int scalar_xor_arena_ptr(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c b/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c deleted file mode 100644 index 6f0efd9202ca..000000000000 --- a/tools/testing/selftests/bpf/progs/verifier_arena_scalar.c +++ /dev/null @@ -1,1155 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include -#include "../../../include/linux/filter.h" -#include "bpf_misc.h" - -void *bpf_arena_alloc_pages(void *map, void *addr, __u32 page_cnt, int node_id, - __u64 flags) __ksym; -void bpf_arena_free_pages(void *map, void *ptr, __u32 page_cnt) __ksym; - -#ifdef __TARGET_ARCH_arm64 -#define ARENA_VM_START (1ull << 32) -#else -#define ARENA_VM_START (1ull << 44) -#endif - -struct { - __uint(type, BPF_MAP_TYPE_ARENA); - __uint(map_flags, BPF_F_MMAPABLE); - __uint(max_entries, 4); - __ulong(map_extra, ARENA_VM_START); -} arena SEC(".maps"); - -struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 1); - __type(key, int); - __type(value, long long); -} hash SEC(".maps"); - -/* JITs that take BPF_F_ARENA_SCALAR */ -#define __arena_scalar __flag(BPF_F_ARENA_SCALAR) __arch_x86_64 __arch_arm64 - -/* BTF FUNC records are not generated for kfuncs referenced from inline assembly */ -void __kfunc_btf_root(void) -{ - bpf_arena_alloc_pages(0, 0, 0, 0, 0); - bpf_arena_free_pages(0, 0, 0); -} - -/* Tests start with r6 = address of a new page as the user space sees it, a number */ - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load and store of every size") -__success __retval(0) -__load_if_JITed() -__naked void ld_st_sizes(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - r1 = 0x1122334455667788 ll; \ - *(u64 *)(r6 + 0) = r1; \ - *(u32 *)(r6 + 8) = r1; \ - *(u16 *)(r6 + 12) = r1; \ - *(u8 *)(r6 + 14) = r1; \ - *(u64 *)(r6 + 16) = 0x1234; \ - *(u32 *)(r6 + 24) = 0x5678; \ - *(u16 *)(r6 + 28) = 0x9a; \ - *(u8 *)(r6 + 30) = 0xbc; \ - r0 = 1; \ - r2 = *(u64 *)(r6 + 0); \ - if r2 != r1 goto 9f; \ - r0 = 2; \ - r2 = *(u32 *)(r6 + 8); \ - if r2 != 0x55667788 goto 9f; \ - r0 = 3; \ - r2 = *(u16 *)(r6 + 12); \ - if r2 != 0x7788 goto 9f; \ - r0 = 4; \ - r2 = *(u8 *)(r6 + 14); \ - if r2 != 0x88 goto 9f; \ - r0 = 5; \ - r2 = *(u64 *)(r6 + 16); \ - if r2 != 0x1234 goto 9f; \ - r0 = 6; \ - r2 = *(u32 *)(r6 + 24); \ - if r2 != 0x5678 goto 9f; \ - r0 = 7; \ - r2 = *(u16 *)(r6 + 28); \ - if r2 != 0x9a goto 9f; \ - r0 = 8; \ - r2 = *(u8 *)(r6 + 30); \ - if r2 != 0xbc goto 9f; \ - /* the address is what it was */ \ - r0 = 9; \ - if r6 != r7 goto 9f; \ - r0 = 10; \ - r7 >>= 32; \ - if r7 == 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load into the register that holds the address") -__success __retval(0) -__load_if_JITed() -__naked void ld_into_base(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 77; \ - *(u64 *)(r6 + 0) = r1; \ - r6 = *(u64 *)(r6 + 0); \ - r0 = 1; \ - if r6 != 77 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: sign extending load") -__success __retval(0) -__load_if_JITed() -__naked void ldsx(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - *(u64 *)(r6 + 0) = 0x80; \ - .8byte %[ldsx_insn]; /* r2 = *(s8 *)(r6 + 0) */ \ - r0 = 1; \ - if r2 != -128 goto 9f; \ - r0 = 2; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(ldsx_insn, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_B, - BPF_REG_2, BPF_REG_6, 0, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: address loaded from arena") -__success __retval(0) -__load_if_JITed() -__naked void ptr_chase(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - /* page[0] = &page[64]; page[64] = 5; */ \ - r1 = r6; \ - r1 += 64; \ - *(u64 *)(r6 + 0) = r1; \ - *(u64 *)(r1 + 0) = 5; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 1; \ - if r2 != r1 goto 9f; \ - r3 = *(u64 *)(r2 + 0); \ - r0 = 2; \ - if r3 != 5 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: atomics") -__success __retval(0) -__load_if_JITed() -__naked void atomics(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - *(u64 *)(r6 + 8) = 1; \ - r1 = 2; \ - lock *(u64 *)(r6 + 8) += r1; \ - r1 = 4; \ - .8byte %[fetch_add_insn]; /* r1 = atomic_fetch_add((u64 *)(r6 + 8), r1) */ \ - r0 = 1; \ - if r1 != 3 goto 9f; \ - r1 = 8; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r6 + 8, r1) */ \ - r0 = 2; \ - if r1 != 7 goto 9f; \ - r0 = 8; \ - r1 = 16; \ - .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r6 + 8, r0, r1) */ \ - r2 = r0; \ - r0 = 3; \ - if r2 != 8 goto 9f; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != 16 goto 9f; \ - r0 = 5; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_6, BPF_REG_1, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching and/or/xor") -__success __retval(0) -__load_if_JITed() -__naked void fetch_and_or_xor(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = 0x11000000110 ll; \ - *(u64 *)(r6 + 8) = r7; \ - r0 = r6; \ - r0 ^= 0x55; \ - r1 = 0x1100000011 ll; \ - .8byte %[fetch_or_dw]; /* r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ - r2 = r6; \ - r2 ^= 0x55; \ - if r0 == r2 goto 1f; \ - r0 = 99; \ - goto 9f; \ -1: r0 = 1; \ - if r1 != r7 goto 9f; \ - r7 = 0x11100000111 ll; \ - r1 = 0x11000000111 ll; \ - .8byte %[fetch_and_dw]; /* r1 = atomic_fetch_and((u64 *)(r6 + 8), r1) */ \ - r0 = 2; \ - if r1 != r7 goto 9f; \ - r7 = 0x11000000111 ll; \ - r1 = 0x101 ll; \ - .8byte %[fetch_xor_dw]; /* r1 = atomic_fetch_xor((u64 *)(r6 + 8), r1) */ \ - r0 = 3; \ - if r1 != r7 goto 9f; \ - r7 = 0x11000000010 ll; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != r7 goto 9f; \ - *(u32 *)(r6 + 16) = 0x110; \ - r1 = 0xdead000000000011 ll; \ - .8byte %[fetch_or_w]; /* r1 = atomic_fetch_or((u32 *)(r6 + 16), r1) */ \ - r0 = 5; \ - if r1 != 0x110 goto 9f; \ - r1 = 0xdead000000000101 ll; \ - .8byte %[fetch_and_w]; /* r1 = atomic_fetch_and((u32 *)(r6 + 16), r1) */ \ - r0 = 6; \ - if r1 != 0x111 goto 9f; \ - r1 = 0xdead000000000011 ll; \ - .8byte %[fetch_xor_w]; /* r1 = atomic_fetch_xor((u32 *)(r6 + 16), r1) */ \ - r0 = 7; \ - if r1 != 0x101 goto 9f; \ - r2 = *(u32 *)(r6 + 16); \ - r0 = 8; \ - if r2 != 0x110 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)), - __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)), - __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 16)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching and/or/xor with the operand in r0") -__success __retval(0) -__load_if_JITed() -__naked void fetch_and_or_xor_r0(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = 0x11000000110 ll; \ - *(u64 *)(r6 + 8) = r7; \ - r0 = 0x1100000011 ll; \ - .8byte %[fetch_or_dw]; /* r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != r7 goto 9f; \ - r7 = 0x11100000111 ll; \ - r0 = 0x11000000111 ll; \ - .8byte %[fetch_and_dw]; /* r0 = atomic_fetch_and((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 2; \ - if r2 != r7 goto 9f; \ - r7 = 0x11000000111 ll; \ - r0 = 0x101 ll; \ - .8byte %[fetch_xor_dw]; /* r0 = atomic_fetch_xor((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 3; \ - if r2 != r7 goto 9f; \ - r7 = 0x11000000010 ll; \ - r2 = *(u64 *)(r6 + 8); \ - r0 = 4; \ - if r2 != r7 goto 9f; \ - *(u32 *)(r6 + 16) = 0x110; \ - r0 = 0xdead000000000011 ll; \ - .8byte %[fetch_or_w]; /* r0 = atomic_fetch_or((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 5; \ - if r2 != 0x110 goto 9f; \ - r0 = 0xdead000000000101 ll; \ - .8byte %[fetch_and_w]; /* r0 = atomic_fetch_and((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 6; \ - if r2 != 0x111 goto 9f; \ - r0 = 0xdead000000000011 ll; \ - .8byte %[fetch_xor_w]; /* r0 = atomic_fetch_xor((u32 *)(r6 + 16), r0) */ \ - r2 = r0; \ - r0 = 7; \ - if r2 != 0x101 goto 9f; \ - r2 = *(u32 *)(r6 + 16); \ - r0 = 8; \ - if r2 != 0x110 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_and_dw, BPF_ATOMIC_OP(BPF_DW, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_xor_dw, BPF_ATOMIC_OP(BPF_DW, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)), - __imm_insn(fetch_or_w, BPF_ATOMIC_OP(BPF_W, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)), - __imm_insn(fetch_and_w, BPF_ATOMIC_OP(BPF_W, BPF_AND | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)), - __imm_insn(fetch_xor_w, BPF_ATOMIC_OP(BPF_W, BPF_XOR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 16)) - : __clobber_all); -} - -/* - * The page is gone, so the access faults. The fetched register has to come back - * 0 and everything else has to be left alone, r0 included. Reading the address - * again has to give 0 as well: had the fault been recovered by the arena mapping - * its zeroed scratch page instead of by the JIT's exception table entry, the - * access would have succeeded and left a 1 there. - */ -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching or through a number to a freed page") -__success __retval(0) -__load_if_JITed() -__naked void fetch_or_freed_page(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x55; \ - r1 = %[arena] ll; \ - r2 = r6; \ - r3 = 1; \ - call %[bpf_arena_free_pages]; \ - r0 = r6; \ - r0 ^= 0x55; \ - r1 = 1; \ - .8byte %[fetch_or_dw]; /* faults: r1 = atomic_fetch_or((u64 *)(r6 + 8), r1) */ \ - r2 = r6; \ - r2 ^= 0x55; \ - if r0 == r2 goto 1f; \ - r0 = 1; \ - goto 9f; \ -1: r0 = 2; \ - if r1 != 0 goto 9f; \ - r3 = *(u64 *)(r6 + 8); \ - r0 = 3; \ - if r3 != 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_arena_free_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_1, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: fetching or with the operand in r0 to a freed page") -__success __retval(0) -__load_if_JITed() -__naked void fetch_or_r0_freed_page(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x55; \ - r1 = %[arena] ll; \ - r2 = r6; \ - r3 = 1; \ - call %[bpf_arena_free_pages]; \ - r0 = 1; \ - .8byte %[fetch_or_dw]; /* faults: r0 = atomic_fetch_or((u64 *)(r6 + 8), r0) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != 0 goto 9f; \ - r3 = *(u64 *)(r6 + 8); \ - r0 = 2; \ - if r3 != 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_arena_free_pages), - __imm_insn(fetch_or_dw, BPF_ATOMIC_OP(BPF_DW, BPF_OR | BPF_FETCH, - BPF_REG_6, BPF_REG_0, 8)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: cmpxchg through r0") -__success __retval(0) -__load_if_JITed() -__naked void cmpxchg_r0(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - /* The address is in r0. The value at the address is not equal to it. */ \ - *(u64 *)(r6 + 0) = 3; \ - r0 = r6; \ - r1 = 5; \ - .8byte %[cmpxchg_insn]; /* r0 = cmpxchg_64(r0 + 0, r0, r1) */ \ - r2 = r0; \ - r0 = 1; \ - if r2 != 3 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 2; \ - if r2 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_0, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: xchg into the register that holds the address") -__success __retval(0) -__load_if_JITed() -__naked void xchg_into_base(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 0) = 3; \ - r1 = r6; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ - r0 = 1; \ - if r1 != 3 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r0 = 2; \ - if r2 != r6 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: xchg into the register that holds a pointer to stack") -__failure __msg("misaligned access off (0x0; 0xffffffffffffffff)+0 size 8") -__naked void xchg_into_stack_ptr(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u64 *)(r10 - 8) = r1; \ - r1 = r10; \ - r1 += -8; \ - .8byte %[xchg_insn]; /* r1 = xchg_64(r1 + 0, r1) */ \ - r0 = 0; \ - exit; \ -" : - : __imm_addr(arena), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_1, 0)) - : __clobber_all); -} - -#ifdef CAN_USE_LOAD_ACQ_STORE_REL - -SEC("syscall") -__arena_scalar -__description("arena_scalar: load-acquire and store-release") -__success __retval(0) -__load_if_JITed() -__naked void load_acq_store_rel(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r7 = r6; \ - r1 = 0x1234; \ - .8byte %[store_release_insn]; /* store_release((u64 *)(r6 + 8), r1) */ \ - .8byte %[load_acquire_insn]; /* r2 = load_acquire((u64 *)(r6 + 8)) */ \ - r0 = 1; \ - if r2 != 0x1234 goto 9f; \ - .8byte %[load_acquire8_insn]; /* w2 = load_acquire((u8 *)(r6 + 8)) */ \ - r0 = 2; \ - if r2 != 0x34 goto 9f; \ - r0 = 3; \ - if r6 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(store_release_insn, - BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_6, BPF_REG_1, 8)), - __imm_insn(load_acquire_insn, - BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)), - __imm_insn(load_acquire8_insn, - BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_6, 8)) - : __clobber_all); -} - -#endif /* CAN_USE_LOAD_ACQ_STORE_REL */ - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number that is not an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void not_in_arena(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - /* nothing is allocated: all loads read 0, stores are dropped */ \ - r6 = 0xdeadbeef00000000 ll; \ - r0 = 1; \ - r2 = *(u64 *)(r6 + 0); \ - if r2 != 0 goto 9f; \ - r0 = 2; \ - r2 = *(u8 *)(r6 - 32768); \ - if r2 != 0 goto 9f; \ - r6 = 0x12345678ffffffff ll; \ - r0 = 3; \ - r2 = *(u64 *)(r6 + 32760); \ - if r2 != 0 goto 9f; \ - *(u64 *)(r6 + 32760) = 1; \ - *(u8 *)(r6 + 32767) = r2; \ - r1 = 1; \ - lock *(u64 *)(r6 + 32760) += r1; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: store through the address of the stack as a number") -__success __retval(0) -__load_if_JITed() -__naked void stack_addr_as_number(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - *(u64 *)(r10 - 8) = 5; \ - r6 = r10; \ - r6 |= 0; \ - /* r6 is a number now. The store goes to arena, not to the stack. */ \ - *(u64 *)(r6 - 8) = 7; \ - r1 = 9; \ - *(u64 *)(r6 - 8) = r1; \ - lock *(u64 *)(r6 - 8) += r1; \ - r2 = *(u64 *)(r10 - 8); \ - r0 = 1; \ - if r2 != 5 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: 64-bit math on the address stays 64-bit") -__success __retval(0) -__load_if_JITed() -__naked void alu64_after_access(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r2 = *(u64 *)(r6 + 0); \ - r7 = r6; \ - r7 += 8; \ - r7 -= r6; \ - r0 = 1; \ - if r7 != 8 goto 9f; \ - r7 = r6; \ - r7 >>= 32; \ - r0 = 2; \ - if r7 == 0 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: st of an immediate changes no register") -__success __retval(0) -__load_if_JITed() -__naked void st_keeps_regs(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r0 = r6; \ - r1 = 0x1111; \ - r2 = 0x2222; \ - *(u64 *)(r0 + 0) = 5; \ - r3 = r0; \ - r0 = 1; \ - if r3 != r6 goto 9f; \ - r0 = 2; \ - if r1 != 0x1111 goto 9f; \ - r0 = 3; \ - if r2 != 0x2222 goto 9f; \ - r0 = 0x3333; \ - r1 = r6; \ - *(u32 *)(r1 + 8) = -7; \ - r3 = r0; \ - r0 = 4; \ - if r3 != 0x3333 goto 9f; \ - r0 = 5; \ - if r1 != r6 goto 9f; \ - r0 = 6; \ - r3 = *(u64 *)(r6 + 0); \ - if r3 != 5 goto 9f; \ - r0 = 7; \ - r3 = *(u64 *)(r6 + 8); \ - r4 = 0xfffffff9 ll; \ - if r3 != r4 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: access through a number with garbage in the upper half") -__success __retval(0) -__load_if_JITed() -__naked void st_value_garbage(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0xdeadbeef00001000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - *(u64 *)(r8 + 0) = 1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - *(u8 *)(r8 + 1) = 1; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - *(u32 *)(r8 + 4) = r1; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r2 = *(u16 *)(r8 + 2); \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 5; \ - if r1 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: access through a number without the upper half") -__success __retval(0) -__load_if_JITed() -__naked void st_value_small(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0x1000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - *(u64 *)(r8 + 0) = 1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - *(u8 *)(r8 + 1) = 1; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - *(u32 *)(r8 + 4) = r1; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r2 = *(u16 *)(r8 + 2); \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 5; \ - if r1 != 3 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: atomics through a number that is not an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void atomic_value(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = 0xdeadbeef00001000 ll; \ - *(u64 *)(r6 + 2048) = r1; \ - r7 = *(u64 *)(r6 + 2048); \ - r8 = *(u64 *)(r6 + 2048); \ - r1 = 3; \ - lock *(u64 *)(r8 + 0) += r1; \ - r0 = 1; \ - if r8 != r7 goto 9f; \ - .8byte %[fetch_add_insn]; \ - r0 = 2; \ - if r8 != r7 goto 9f; \ - .8byte %[xchg_insn]; \ - r0 = 3; \ - if r8 != r7 goto 9f; \ - r0 = 0; \ - .8byte %[cmpxchg_insn]; \ - r0 = 4; \ - if r8 != r7 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(fetch_add_insn, BPF_ATOMIC_OP(BPF_DW, BPF_ADD | BPF_FETCH, - BPF_REG_8, BPF_REG_1, 0)), - __imm_insn(xchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_8, BPF_REG_1, 0)), - __imm_insn(cmpxchg_insn, BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, BPF_REG_8, BPF_REG_1, 0)) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number and pointer to arena at the same insn") -__success __retval(0) -__load_if_JITed() -__naked void mixed_number_arena(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r6 + 8) = 0x1234; \ - /* a number that the verifier does not know, 0 at run time */ \ - r7 = *(u64 *)(r6 + 16); \ - r8 = r6; \ - if r7 != 0 goto 1f; \ - .8byte %[cast_kern_insn]; \ -1: *(u8 *)(r8 + 0) = 1; \ - *(u8 *)(r8 + 1) = r7; \ - r2 = *(u8 *)(r8 + 1); \ - lock *(u64 *)(r8 + 24) += r7; \ - r0 = 0; \ - if r7 != 0 goto 9f; \ - /* pointer to arena only: JIT adds all 64 bits of r8 to the base */ \ - r0 = 2; \ - r2 = *(u64 *)(r8 + 8); \ - if r2 != 0x1234 goto 9f; \ - r0 = 3; \ - r2 = *(u8 *)(r6 + 0); \ - if r2 != 1 goto 9f; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm_insn(cast_kern_insn, BPF_RAW_INSN(BPF_ALU64 | BPF_MOV | BPF_X, - BPF_REG_8, BPF_REG_8, 1, 1)) - : __clobber_all); -} - -SEC("syscall") -__description("arena_scalar: no flag, no access through a number") -__failure __msg("R6 invalid mem access 'scalar'") -__naked void no_flag(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r6 = 0x100000000000 ll; \ - r0 = *(u64 *)(r6 + 0); \ - exit; \ -" : - : __imm_addr(arena) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: pointer that is NULL is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void null_ptr(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u32 *)(r10 - 4) = r1; \ - r2 = r10; \ - r2 += -4; \ - r1 = %[hash] ll; \ - call %[bpf_map_lookup_elem]; \ - r1 = r0; \ - r0 = 1; \ - if r1 != 0 goto 9f; \ - /* nothing is allocated: the load reads 0 */ \ - r0 = *(u64 *)(r1 + 0); \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: pointer that is NULL with an offset is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void null_ptr_off(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r1 = 0; \ - *(u32 *)(r10 - 4) = r1; \ - r2 = r10; \ - r2 += -4; \ - r1 = %[hash] ll; \ - call %[bpf_map_lookup_elem]; \ - r1 = r0; \ - r0 = 1; \ - if r1 != 0 goto 9f; \ - r1 += 8; \ - *(u64 *)(r1 + 0) = 5; \ - r0 = *(u64 *)(r1 + 0); \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number that is less than a page is an address in arena") -__success __retval(0) -__load_if_JITed() -__naked void small_number(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - call %[bpf_get_prandom_u32]; \ - r1 = r0; \ - r1 &= 0xfff; \ - r0 = *(u64 *)(r1 + 0); \ - exit; \ -" : - : __imm_addr(arena), - __imm(bpf_get_prandom_u32) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number is not a pointer for a helper") -__failure __msg("R2 type=scalar expected=") -__naked void helper_arg(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - r1 = %[hash] ll; \ - r2 = r6; \ - call %[bpf_map_lookup_elem]; \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm_addr(hash), - __imm(bpf_arena_alloc_pages), - __imm(bpf_map_lookup_elem) - : __clobber_all); -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: number and pointer to stack at the same insn") -__failure __msg("same insn cannot be used with different pointers") -__load_if_JITed() -__naked void mixed_number_stack(void) -{ - asm volatile (" \ - r1 = %[arena] ll; \ - r2 = 0; \ - r3 = 1; \ - r4 = -1; \ - r5 = 0; \ - call %[bpf_arena_alloc_pages]; \ - r6 = r0; \ - r0 = 100; \ - if r6 == 0 goto 9f; \ - *(u64 *)(r10 - 8) = 0; \ - call %[bpf_get_prandom_u32]; \ - if w0 != 0 goto 1f; \ - r6 = r10; \ - r6 += -8; \ -1: r0 = *(u64 *)(r6 + 0); \ - r0 = 0; \ -9: exit; \ -" : - : __imm_addr(arena), - __imm(bpf_arena_alloc_pages), - __imm(bpf_get_prandom_u32) - : __clobber_all); -} - -static int st_cb(__u64 idx, void *ctx) -{ - volatile long *p = *(volatile long **)ctx; - - p[idx] = 7; - return 0; -} - -SEC("syscall") -__arena_scalar -__description("arena_scalar: store through a number in a callback") -__success __retval(0) -__load_if_JITed() -int st_in_callback(void *unused) -{ - volatile long *p = bpf_arena_alloc_pages(&arena, NULL, 1, -1, 0); - - if (!p) - return 100; - bpf_loop(4, st_cb, &p, 0); - return p[0] + p[3] + p[4] - 14; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index df8d5309657e..05efa9f4a522 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -2159,6 +2159,47 @@ l0_%=: r0 = 0; \ : __clobber_all); } +/* + * The insn that forks is the head of a loop that scalar evolution counted. The state with + * [-1, 0] is kept there, and the state with 0 that is split off is within it. + */ +SEC("socket") +__description("maybe_fork_scalars: AND at the head of a counted loop rejects OOB") +__failure __msg("invalid access to map value") +__naked void and_scalar_fork_at_long_loop_head_rejects_oob(void) +{ + asm volatile (" \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r9 = r0; \ + r7 = 0; \ + r6 = -1; \ +l1_%=: r6 &= 8; \ + /* r6 is 8 (current) or 0 (pushed) */ \ + r6 ^= 8; \ + /* pushed path: r6 = 8, OOB for value_size=8 */ \ + r1 = r9; \ + r1 += r6; \ + r0 = *(u8*)(r1 + 0); \ + r1 = *(u64*)(r9 + 0); \ + r6 = -1; \ + if r1 == 0 goto l2_%=; \ + r6 = 0; \ +l2_%=: r7 += 1; \ + if r7 < 100 goto l1_%=; \ +l0_%=: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + /* * Last jump can be detected as always taken because the intersection of R5 and * R7 32bit tnums produces a constant that isn't within R7's s32 bounds. diff --git a/tools/testing/selftests/bpf/progs/verifier_scev.c b/tools/testing/selftests/bpf/progs/verifier_scev.c index 661ce9ca8f93..fa01a616af9d 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scev.c +++ b/tools/testing/selftests/bpf/progs/verifier_scev.c @@ -1377,6 +1377,40 @@ __naked __noinline __used static int probe_read_stack(void) : __clobber_all); } +/* + * r3 = r8 + 4 is linked to r8 when it enters the loop. Widening drops the link. If + * it leaves the 4 behind then r4 = r3 + 1 is taken for r3 - 3 when r3 is compared: + * r4 <= 7 instead of r4 <= 11, and the read of r9 looks like dead code. + */ +SEC("socket") +__failure __msg("R9 !read_ok") +__naked void widened_reg_is_not_linked(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r8 = r0;" + "r8 &= 0xff;" + "r3 = r8;" + "r3 += 4;" + "r1 = 0;" +"1:" + "r4 = r3;" + "r4 += 1;" + "if r3 > 10 goto 3f;" + "if r4 <= 7 goto 2f;" + "r0 = *(u64 *)(r9 + 0);" +"2:" + "r3 += 1;" + "r1 += 1;" + "if r1 < 2 goto 1b;" +"3:" + "r0 = r8;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + /* The unknown SCEV of an unbound STACK_MISC slot must not prevent widening. */ SEC("tracepoint") __success @@ -1592,7 +1626,7 @@ __msg_next(" r0=(+ r0 -1) / (linear (+ r0 -1) -1)") __msg("loop header at 1, header_count is 3 ") __msg("loop header at 1, widening r0 to 1..3 step 1") __msg("1: R0=scalar(smin=umin=smin32=umin32=1,smax=umax=smax32=umax32=3,var_off=(0x0; 0x3)) loop_stack=1") -__msg("loop header at 1, clamping r0 to 1..2 (s32 1..2) step 1") +__msg("loop header at 1, clamping r0 to 1..3 step 1") __msg("processed 5 insns") __naked void post_cond_jne_neg_step(void) { @@ -1604,6 +1638,61 @@ __naked void post_cond_jne_neg_step(void) " ::: __clobber_all); } +/* + * r1 and r3 are widened independently, so the r3 == 5 path reaches the back + * edge with r3 past the last value it takes at the header. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 5 ") +__msg("loop header at 2, widening r1 to 0..4 step 1") +__msg("loop header at 2, widening r3 to 0..4 step 1") +__msg("3: (55) if r3 != 0x5 goto pc+1 {{.*}}; R3=5") +__msg("loop header at 2, clamping r3 to 0..4 step 1") +__msg("from 6 to 2: safe") +__naked void clamp_dependent_overshoot(void) +{ + asm volatile (" \ + r1 = 0; \ + r3 = 0; \ +1: r3 += 1; \ + if r3 != 5 goto 2f; \ + r2 = 0; \ +2: r1 += 1; \ + if r1 != 5 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + +/* Same with a 32-bit dependent register, SCEV (zext32 (linear r3 1)) */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 2, header_count is 5 ") +__msg("loop header at 2, widening r1 to 0..4 step 1") +__msg("loop header at 2, widening r3 to 0..4 step 1") +__msg("3: (56) if w3 != 0x5 goto pc+1 {{.*}}; R3=5") +__msg("loop header at 2, clamping r3 to 0..4 step 1") +__msg("from 6 to 2: safe") +__naked void clamp_dependent_overshoot32(void) +{ + asm volatile (" \ + r1 = 0; \ + w3 = 0; \ +1: w3 += 1; \ + if w3 != 5 goto 2f; \ + r2 = 0; \ +2: r1 += 1; \ + if r1 != 5 goto 1b; \ + r0 = 0; \ + exit; \ +" ::: __clobber_all); +} + SEC("xdp") __success __log_level(2) @@ -1771,7 +1860,7 @@ __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("loop header at 1, header_count is 4") -__msg("loop header at 1, widening r0 to 0..3 (s32 0..3) step 1") +__msg("loop header at 1, widening r0 to 0..3 step 1") __msg("loop header at 1, clamping r0") __msg("4: R0=3") __naked void pre_cond_je32(void) @@ -2512,6 +2601,104 @@ __naked void no_widen_any_stack_origin(void) " ::: __clobber_all); } +__naked __noinline __used static int scrub_slot_callee(void) +{ + asm volatile (" \ + *(u32 *)(r1 + 4) = 1; \ + call %[bpf_get_prandom_u32]; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * The callee turns the caller's fp-8 into misc, so the states at the call site + * differ between the first and the second iteration, while the states inside + * the callee are identical. Verification fails with an infinite loop error if + * states.c:current_loop() does not search upper frames for active loops. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__msg("12: safe") +__naked void clamp_callee_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: r1 = r10; \ + r1 += -8; \ + call scrub_slot_callee; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + +/* + * The inner loop turns fp-8 from a spill into misc. On the next outer + * iteration, the states inside the inner loop match its first invocation. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 4, widening r7 to 0..2 step 1") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__naked void clamp_nested_loop_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: r7 = 0; \ +2: *(u32 *)(r10 - 4) = 1; \ + r7 += 1; \ + if r7 < 3 goto 2b; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + +/* + * Same as above, but the inner loop is not recognized by SCEV and is unrolled + * with a precise r7. The states at its header still match between the outer + * iterations and must not be reported as an infinite loop: the path between + * them crosses the back edge of the terminating outer loop. + */ +SEC("socket") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg("loop header at 3, widening r6 to 0..9 step 1") +__msg("loop header at 5, can't compute iterations count") +__msg("loop header at 3, clamping r6 to 0..9 step 1") +__naked void clamp_nested_unrolled_loop_state_pruning(void) +{ + asm volatile (" \ + w1 = 1; \ + *(u32 *)(r10 - 8) = r1; \ + r6 = 0; \ +1: *(u32 *)(r10 - 4) = 1; \ + r7 = 1; \ +2: r7 *= 2; \ + if r7 < 4 goto 2b; \ + r6 += 1; \ + if r6 < 10 goto 1b; \ + r0 = *(u32 *)(r10 - 8); \ + exit; \ +" ::: __clobber_all); +} + /* * r7 starts as 1 + 6*k and invariant r8 as 1 + 9*k (0 <= k <= 3). * The (any r7 r8) union must preserve base 1 and step gcd(6, 9) = 3. @@ -2971,6 +3158,115 @@ __naked void ptr_stack_other_frame(void) " ::: __clobber_all); } +SEC("xdp") +__success +__log_level(2) +__msg("scev: incompatible latch operands r6 (map_value), r7 (map_value)") +__msg("loop header at 11, can't compute iterations count at 14") +__naked void ptr_map_value_jmp32(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" r6 += 1; \ + if w6 < w7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__failure +__log_level(2) +__msg("loop header at 11, can't compute iterations count") +__msg("R6 32-bit pointer arithmetic prohibited") +__naked void ptr_map_value_alu32_latch(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" w6 += 1; \ + if r6 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 11, can't compute iterations count") +__naked void ptr_map_value_alu32_latch2(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r6 = r0; \ + r7 = r6; \ + r7 += 4; \ + r9 = 0; \ +1:" LIMIT_ITERATIONS \ +" r6 += 1; \ + w8 = w6; \ + if r8 < r7 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + +SEC("xdp") +__success +__log_level(2) +__msg("loop header at 8, can't widen r0, expr is (zext32 (+ r0 1))") +__naked void ptr_map_value_alu32_non_latch(void) +{ + asm volatile (" \ + *(u64 *)(r10 - 8) = 0; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 2f; \ + r1 = 0; \ +1: r1 += 1; \ + w0 = w0; \ + w0 += 1; \ + if r1 < 4 goto 1b; \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), __imm_addr(map) + : __clobber_all); +} + /* Metadata and packet data have different origins. */ SEC("xdp") __success @@ -3221,21 +3517,23 @@ __naked void ptr_mem_other_reservation(void) : __clobber_all); } +/* An access to a buffer at a variable offset is rejected: the pointer must not be widened. */ SEC("iter/bpf_map_elem") __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) __msg("header_count is 4 ") -__msg("widening r6 to 0..3 step 1") +__msg("can't widen r6, expr is (+ r6 8)") __naked void ptr_buf(void) { asm volatile (" \ r6 = *(u64 *)(r1 + %[value]); \ if r6 == 0 goto 2f; \ - r7 = r6; \ - r7 += 4; \ -1: r6 += 1; \ - if r6 < r7 goto 1b; \ + r7 = 0; \ +1: r0 = *(u32 *)(r6 + 0); \ + r6 += 8; \ + r7 += 1; \ + if r7 < 4 goto 1b; \ 2: r0 = 0; \ exit; \ " : @@ -3351,7 +3649,7 @@ SEC("xdp") __failure __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) -__msg("infinite loop detected at insn 6") +__msg("The sequence of 8193 jumps is too complex.") __naked void nested_terminating_prune(void) { asm volatile (" \ @@ -3448,4 +3746,194 @@ __naked void dominating_exit_bounds_count(void) " ::: __clobber_all); } +/* + * The counter is copied to a callee-saved register for the call and the copy is compared. + */ +SEC("xdp") +__success +__log_level(2) +__flag(BPF_F_TEST_STATE_FREQ) +__msg(" scev at latch 5:") +__msg(" r7=(zext32 r1) / (zext32 (linear r1 2))") +__msg("loop header at 1, header_count is 100 ") +__msg("loop header at 1, widening r1 to 0..198 step 2") +__naked void post_cond_jlt32_copy(void) +{ + asm volatile (" \ + w1 = 0; \ +1: w7 = w1; \ + call %[bpf_get_prandom_u32]; \ + w1 = w7; \ + w1 += 2; \ + if w7 < 198 goto 1b; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* + * r6 is a pointer to ringbuf memory with a step of 1. bpf_ringbuf_discard() in the first iteration + * makes a number of it. The offsets 0..1 of the pointer are not the range of this number: it's an + * address. The flag in the 32-bit spill is not widened, so the second iteration is verified + * on its own. + */ +SEC("socket") +__failure __msg("math between map_value pointer and register with unbounded min value") +__naked void clamp_released_pointer(void) +{ + asm volatile ( + "r1 = 0;" + "*(u32 *)(r10 - 12) = r1;" + "r2 = r10;" + "r2 += -12;" + "r1 = %[small_map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 9f;" + "r8 = r0;" + "r1 = %[ringbuf] ll;" + "r2 = 16;" + "r3 = 0;" + "call %[bpf_ringbuf_reserve];" + "if r0 == 0 goto 9f;" + "r6 = r0;" + "r7 = r0;" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + "r1 = r8;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" + "goto 3f;" +"2:" + "r1 = r7;" + "r2 = 0;" + "call %[bpf_ringbuf_discard];" +"3:" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 1;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), __imm(bpf_ringbuf_reserve), __imm(bpf_ringbuf_discard), + __imm_addr(small_map), __imm_addr(ringbuf) + : __clobber_all); +} + +/* The same with a packet pointer and bpf_skb_pull_data() */ +SEC("tc") +__failure __msg("math between map_value pointer and register with unbounded min value") +__naked void clamp_invalidated_pkt_pointer(void) +{ + asm volatile ( + "r7 = r1;" + "r1 = 0;" + "*(u32 *)(r10 - 12) = r1;" + "r2 = r10;" + "r2 += -12;" + "r1 = %[small_map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 9f;" + "r8 = r0;" + "r6 = *(u32 *)(r7 + %[data]);" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + "r1 = r8;" + "r1 += r6;" + "r0 = *(u8 *)(r1 + 0);" + "goto 3f;" +"2:" + "r1 = r7;" + "r2 = 0;" + "call %[bpf_skb_pull_data];" +"3:" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 1;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), __imm(bpf_skb_pull_data), __imm_addr(small_map), + __imm_const(data, offsetof(struct __sk_buff, data)) + : __clobber_all); +} + +/* + * The compare with data_end is for r6 of this iteration: 100 bytes from r6 are in the packet. Then + * r6 moves by 100. The range that the compare found is not for the r6 that returns to the head. + */ +SEC("tc") +__failure __msg("invalid access to packet") +__naked void clamp_pkt_pointer_keeps_no_range(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[data_end]);" + "r6 = *(u32 *)(r1 + %[data]);" + "r9 = 0;" + "w1 = 0;" + "*(u32 *)(r10 - 8) = r1;" +"1:" + "w4 = *(u32 *)(r10 - 8);" + "if w4 == 0 goto 2f;" + /* second iteration: nothing compared this r6 with data_end */ + "r0 = *(u64 *)(r6 + 56);" +"2:" + "r5 = r6;" + "r5 += 100;" + "if r5 > r7 goto 9f;" + "w4 = 1;" + "*(u32 *)(r10 - 8) = r4;" + "r6 += 100;" + "r9 += 1;" + "if r9 < 2 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + +/* A compare in front of the loop is for all iterations: the range at the entry stays */ +SEC("tc") +__success __retval(0) +__naked void clamp_pkt_pointer_keeps_range_of_entry(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[data_end]);" + "r6 = *(u32 *)(r1 + %[data]);" + "r5 = r6;" + "r5 += 40;" + "if r5 > r7 goto 9f;" + "r9 = 0;" +"1:" + "r0 = *(u64 *)(r6 + 0);" + "r6 += 8;" + "r9 += 1;" + "if r9 < 4 goto 1b;" +"9:" + "r0 = 0;" + "exit;" + : + : __imm_const(data, offsetof(struct __sk_buff, data)), + __imm_const(data_end, offsetof(struct __sk_buff, data_end)) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c index b9f799f6e405..15a12497e074 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg.c @@ -433,6 +433,125 @@ __naked void stack_arg_sequential_calls(void) ); } +/* + * A JIT passes the first stack arguments in registers. may_goto is not a call: they must be + * the same after it, also when it looks at the clock. That is once in 65535 iterations. + */ +__naked __noinline __used +static long loops_then_reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "r1 = 0;" + "1:" + "may_goto 2f;" + "r1 += 1;" + "if r1 < 70000 goto 1b;" + "2:" + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: may_goto looks at the clock, then the callee reads its args") +__success __retval(876) +__naked void stack_arg_may_goto_in_callee(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "call loops_then_reads_8args;" + "exit;" + ::: __clobber_all + ); +} + +__naked __noinline __used +static long spins_then_reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: the time of may_goto is up, then the callee reads its args") +__success __retval(876) +__naked void stack_arg_may_goto_timeout_in_callee(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "call spins_then_reads_8args;" + "exit;" + ::: __clobber_all + ); +} + +__naked __noinline __used +static long reads_8args(long a, long b, long c, long d, long e, long f, long g, long h) +{ + asm volatile ( + "r0 = *(u64 *)(r11 + 8);" + "r1 = *(u64 *)(r11 + 16);" + "r0 += r1;" + "r1 = *(u64 *)(r11 + 24);" + "r0 += r1;" + "exit;" + ); +} + +SEC("tc") +__description("stack_arg: may_goto looks at the clock between the stores of the args and the call") +__success __retval(876) +__naked void stack_arg_may_goto_in_caller(void) +{ + asm volatile ( + "*(u64 *)(r11 - 24) = 800;" + "*(u64 *)(r11 - 16) = 70;" + "*(u64 *)(r11 - 8) = 6;" + "r6 = 0;" + "1:" + "may_goto 2f;" + "r6 += 1;" + "if r6 < 70000 goto 1b;" + "2:" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call reads_8args;" + "exit;" + ::: __clobber_all + ); +} + #else SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c index ab1955852233..5bfcd300c7a0 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c @@ -5,6 +5,17 @@ #include #include "bpf_misc.h" +struct timer_value { + struct bpf_timer timer; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct timer_value); +} timer_map SEC(".maps"); + #if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) || \ (defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64)) && \ defined(__BPF_FEATURE_STACK_ARGUMENT) @@ -130,6 +141,25 @@ static void subprog_bad_ptr_7args(long *a, int b, int c, int d, int e, int f, in ); } +__noinline __used __naked +static int callback_6args(long a, long b, long c, long d, long e, long f) +{ + asm volatile ("r0 = *(u64 *)(r11 + 8);" + "exit;" + ::: __clobber_all + ); +} + +__noinline __used __naked +static void subprog_call_bad_ptr_7args(void) +{ + asm volatile ("*(u64 *)(r11 - 8) = 0;" + "call subprog_bad_ptr_7args;" + "exit;" + ::: __clobber_all + ); +} + SEC("tc") __description("stack_arg: read without caller write") __failure @@ -152,7 +182,7 @@ __naked void stack_arg_read_without_write_1(void) SEC("tc") __description("stack_arg: read with not-initialized caller write") __failure -__msg("R0 !read_ok") +__msg("invalid read from uninitialized stack arg off 8") __btf_func_path("btf__verifier_stack_arg_order.bpf.o") __naked void stack_arg_read_without_write_2(void) { @@ -171,6 +201,95 @@ __naked void stack_arg_read_without_write_2(void) ); } +SEC("tc") +__description("stack_arg: unreliable callee with too few stack args") +__failure +__msg("invalid read from uninitialized stack arg off 16") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_read_with_short_bridge_write(void) +{ + asm volatile ("r1 = 0;" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + "*(u64 *)(r11 - 8) = 0;" + "*(u64 *)(r11 - 16) = 0;" + "call subprog_bad_ptr_7args;" + "call subprog_call_bad_ptr_7args;" + "exit;" + ::: __clobber_all + ); +} + +SEC("tc") +__description("stack_arg: synchronous callback with stack args") +__failure +__msg("callback subprog cannot have stack args") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_sync_callback(void) +{ + asm volatile ("r6 = 0;" + "*(u64 *)(r11 - 8) = r6;" + "r1 = 1;" + "r2 = %[callback_6args];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call callback_6args;" + "r0 = 0;" + "exit;" + : + : __imm_ptr(callback_6args), + __imm(bpf_loop) + : __clobber_common, "r6" + ); +} + +SEC("tc") +__description("stack_arg: asynchronous callback with stack args") +__failure +__msg("callback subprog cannot have stack args") +__btf_func_path("btf__verifier_stack_arg_order.bpf.o") +__naked void stack_arg_async_callback(void) +{ + asm volatile ("r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r1 = %[timer_map] ll;" + "r2 = r10;" + "r2 += -4;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l0_%=;" + "r6 = r0;" + "r7 = 0;" + "*(u64 *)(r11 - 8) = r7;" + "r1 = r6;" + "r2 = %[callback_6args];" + "call %[bpf_timer_set_callback];" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "*(u64 *)(r11 - 8) = 6;" + "call callback_6args;" + "l0_%=: r0 = 0;" + "exit;" + : + : __imm_addr(timer_map), + __imm_ptr(callback_6args), + __imm(bpf_map_lookup_elem), + __imm(bpf_timer_set_callback) + : __clobber_common, "r6", "r7" + ); +} + #else SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c index 7fa7bc207850..e6a9021b197b 100644 --- a/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_stack_ptr.c @@ -588,35 +588,37 @@ __naked void stack_check_size_512_with_may_goto(void) /* * Verify that old PTR_TO_STACK state is considered a super-set of - * new PTR_TO_STACK state when new variable range is a sub-range - * of the old range, e.g. old [-72, -16] vs new [-64, -16]. + * new PTR_TO_STACK state when the new offset is within the old range, + * e.g. old [-72, -16] vs new {-16}. */ SEC("socket") __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) /* - * r7 is widened to [-72, -16] at the loop header (insn 3), - * the loop body sees [-64, -8] after 'r7 += 8' + * r7 is widened to [-72, -16] at the loop header (insn 5), + * the loop body sees [-64, -8] after 'r7 += 8'. */ -__msg("loop header at 3, widening r7 to -72..-16 step 8") +__msg("loop header at 5, widening r7 to -72..-16 step 1") __msg("R7=fp(smin=smin32=-64,smax=smax32=-8") -/* - * back-edge state is clamped to the remaining iterations and pruned - * at the header, because [-64, -16] is within the widened [-72, -16] - */ -__msg("loop header at 3, clamping r7 to -64..-16 (s32 -64..-8) step 8") -__msg("from 5 to 3: safe") -__msg("processed 9 insns") +/* 'r7 = r8' sets r7 to fp-16; its ANY SCEV is not clamped. */ +__msg("R7=fp-16 R8=fp-16 loop_stack=5") +__not_msg("clamping r7") +/* back-edge state is pruned because {-16} is within [-72, -16]. */ +__msg("from 8 to 5: safe") +__msg("processed 12 insns") __naked void stack_ptr_subrange_in_loop(void) { asm volatile (" \ r7 = r10; \ r7 += -72; \ + r8 = r10; \ + r8 += -16; \ r6 = 0; \ - r7 += 8; \ +1: r7 += 8; \ + r7 = r8; \ r6 += 1; \ - if r6 < 8 goto -3; \ + if r6 < 8 goto 1b; \ r0 = 0; \ exit; \ " ::: __clobber_all); diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c b/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c index 55398c04290a..25f37a3dd7cf 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_accept.c @@ -11,6 +11,23 @@ static char buf[64]; +SEC("lsm.s/file_open") +__success +int BPF_PROG(inode_field_before_file_put) +{ + struct file *file; + ino_t ino; + + file = bpf_get_task_exe_file(bpf_get_current_task_btf()); + if (!file) + return 0; + + ino = file->f_inode->i_ino; + bpf_put_file(file); + __sink(ino); + return 0; +} + SEC("lsm.s/file_open") __success int BPF_PROG(get_task_exe_file_and_put_kfunc_from_current_sleepable) diff --git a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c index b5f456d57669..ed2bd2faebf3 100644 --- a/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c +++ b/tools/testing/selftests/bpf/progs/verifier_vfs_reject.c @@ -12,6 +12,25 @@ static char buf[PATH_MAX]; +SEC("lsm.s/file_open") +__failure __msg("invalid mem access 'scalar'") +int BPF_PROG(inode_field_after_file_put) +{ + struct inode *inode; + struct file *file; + ino_t ino; + + file = bpf_get_task_exe_file(bpf_get_current_task_btf()); + if (!file) + return 0; + + inode = file->f_inode; + bpf_put_file(file); + ino = inode->i_ino; + __sink(ino); + return 0; +} + SEC("lsm.s/file_open") __failure __msg("Possibly NULL pointer passed to trusted R1") int BPF_PROG(get_task_exe_file_kfunc_null) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 6380f52f4dd2..76fbe64755ef 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -281,6 +281,27 @@ __bpf_kfunc void bpf_iter_testmod_seq_destroy(struct bpf_iter_testmod_seq *it) it->cnt = 0; } +__bpf_kfunc int bpf_iter_testmod_current_new(struct bpf_iter_testmod_current *it) +{ + it->remaining = 2; + return 0; +} + +__bpf_kfunc struct task_struct * +bpf_iter_testmod_current_next(struct bpf_iter_testmod_current *it) +{ + if (!it->remaining) + return NULL; + it->remaining--; + /* current remains alive independently of the iterator. */ + return current; +} + +__bpf_kfunc void bpf_iter_testmod_current_destroy(struct bpf_iter_testmod_current *it) +{ + it->remaining = 0; +} + __bpf_kfunc void bpf_kfunc_common_test(void) { } @@ -1016,6 +1037,10 @@ BTF_ID_FLAGS(func, bpf_iter_testmod_seq_new, KF_ITER_NEW) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_next, KF_ITER_NEXT | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_iter_testmod_seq_value) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_new, KF_ITER_NEW) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_next, + KF_ITER_NEXT | KF_RET_NULL | KF_ITER_RET_INDEPENDENT) +BTF_ID_FLAGS(func, bpf_iter_testmod_current_destroy, KF_ITER_DESTROY) BTF_ID_FLAGS(func, bpf_kfunc_common_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_arg_test) BTF_ID_FLAGS(func, bpf_kfunc_arena_cap_test) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h index 4ba9def74404..aef780261ef4 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.h @@ -31,6 +31,10 @@ struct bpf_iter_testmod_seq { int cnt; }; +struct bpf_iter_testmod_current { + u64 remaining; +}; + struct bpf_testmod_ops { int (*test_1)(void); void (*test_2)(int a, int b); diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 3b810f1e438f..e9902e651d72 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -33,6 +33,11 @@ struct prog_test_ref_kfunc { #endif struct bpf_iter_testmod_seq; +struct bpf_iter_testmod_current; + +int bpf_iter_testmod_current_new(struct bpf_iter_testmod_current *it) __ksym; +struct task_struct *bpf_iter_testmod_current_next(struct bpf_iter_testmod_current *it) __ksym; +void bpf_iter_testmod_current_destroy(struct bpf_iter_testmod_current *it) __ksym; struct prog_test_pass1 { int x0; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 824e5912ebcb..d24e4fedae46 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -590,8 +590,6 @@ static int parse_test_spec(struct test_loader *tester, update_flags(&spec->prog_flags, BPF_F_XDP_HAS_FRAGS, clear); } else if (strcmp(val, "BPF_F_TEST_REG_INVARIANTS") == 0) { update_flags(&spec->prog_flags, BPF_F_TEST_REG_INVARIANTS, clear); - } else if (strcmp(val, "BPF_F_ARENA_SCALAR") == 0) { - update_flags(&spec->prog_flags, BPF_F_ARENA_SCALAR, clear); } else /* assume numeric value */ { err = parse_int(val, &flags, "test prog flags"); if (err)