diff --git a/arch/arm64/net/bpf_timed_may_goto.S b/arch/arm64/net/bpf_timed_may_goto.S index 1b4a4f91ffc9..f8732b13bb1e 100644 --- a/arch/arm64/net/bpf_timed_may_goto.S +++ b/arch/arm64/net/bpf_timed_may_goto.S @@ -17,16 +17,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) str x7, [sp, #80] /* - * Stack depth was passed in BPF_REG_AX (x9), add it to the BPF_FP - * (x25) to get the pointer to count and timestamp and pass it as the - * first argument in x0. - * - * Before generating the call to arch_bpf_timed_may_goto, the verifier - * generates a load instruction using FP, i.e. REG_AX = *(u64 *)(FP - - * stack_off_cnt), so BPF_REG_FP (x25) is always set up by the arm64 - * jit in this case. + * The pointer to count and timestamp was passed in BPF_REG_AX (x9), + * pass it as the first argument in x0. */ - add x0, x9, x25 + mov x0, x9 bl bpf_check_timed_may_goto /* BPF_REG_AX(x9) will be stored into count, so move return value to it. */ mov x9, x0 diff --git a/arch/loongarch/net/bpf_timed_may_goto.S b/arch/loongarch/net/bpf_timed_may_goto.S index fa128acdb108..4aaca029404b 100644 --- a/arch/loongarch/net/bpf_timed_may_goto.S +++ b/arch/loongarch/net/bpf_timed_may_goto.S @@ -22,14 +22,10 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) st.d a4, sp, 48 /* - * BPF_REG_AX (t0) holds the offset passed in by the verifier; - * add it to BPF_REG_FP (s4) to get the pointer to the count and - * timestamp, then pass it as the first argument in a0. - * - * The verifier emits a load using FP right before this call, - * so BPF_REG_FP (s4) is always set up by the JIT in this case. + * BPF_REG_AX (t0) holds the pointer to the count and timestamp; + * pass it as the first argument in a0. */ - add.d a0, t0, s4 + move a0, t0 bl bpf_check_timed_may_goto /* BPF_REG_AX (t0) will be stored into count, so move the return value to it. */ move t0, a0 diff --git a/arch/powerpc/net/bpf_timed_may_goto.S b/arch/powerpc/net/bpf_timed_may_goto.S index 84ecf6fa7f5d..b8ae9885eb31 100644 --- a/arch/powerpc/net/bpf_timed_may_goto.S +++ b/arch/powerpc/net/bpf_timed_may_goto.S @@ -8,7 +8,7 @@ * arch_bpf_timed_may_goto() trampoline for powerpc64 * * Custom BPF convention (verifier/JIT): - * - input: stack offset in BPF_REG_AX (r12) + * - input: pointer to count and timestamp in BPF_REG_AX (r12) * - output: updated count in BPF_REG_AX (r12) * * Call bpf_check_timed_may_goto(ptr) with normal powerpc64 ABI: @@ -31,11 +31,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) std r7, 64(r1) std r8, 72(r1) - /* - * r3 = BPF_REG_FP + BPF_REG_AX - * BPF_REG_FP is r31; BPF_REG_AX is r12 (stack offset in bytes). - */ - add r3, r31, r12 + /* r3 = BPF_REG_AX (r12) */ + mr r3, r12 bl CFUNC(bpf_check_timed_may_goto) /* Put return value back into AX */ diff --git a/arch/riscv/net/bpf_timed_may_goto.S b/arch/riscv/net/bpf_timed_may_goto.S index 6ec37f634aff..2ce1c89c913e 100644 --- a/arch/riscv/net/bpf_timed_may_goto.S +++ b/arch/riscv/net/bpf_timed_may_goto.S @@ -6,13 +6,11 @@ /* * Trampoline for the BPF timed may_goto loop bound. Custom calling convention: - * - input: stack offset in BPF_REG_AX (t0) + * - input: pointer to count and timestamp in BPF_REG_AX (t0) * - output: updated count in BPF_REG_AX (t0) * - * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI, where - * ptr = BPF_REG_FP (s5) + BPF_REG_AX (t0). BPF R0-R5 (t6, a0-a4) and stack - * arguments 6-8 (a5-a7) are saved across the call; BPF_REG_FP (s5) is - * callee-saved and needs no saving. + * Calls bpf_check_timed_may_goto(ptr) with the standard RISC-V ABI. BPF R0-R5 + * (t6, a0-a4) and stack arguments 6-8 (a5-a7) are saved across the call. */ SYM_FUNC_START(arch_bpf_timed_may_goto) @@ -33,7 +31,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) REG_S a6, 7*SZREG(sp) REG_S a7, 8*SZREG(sp) - add a0, t0, s5 + mv a0, t0 call bpf_check_timed_may_goto mv t0, a0 diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 83da3a00db7e..3ab880fc17a6 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1005,6 +1005,17 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, (jit->prg - patch_brc) >> 1; break; } + if (insn_is_mov_percpu_addr(insn)) { + const u64 *percpu_offset = &get_lowcore()->percpu_offset; + + if (dst_reg != src_reg) + /* lgr %dst,%src */ + EMIT4(0xb9040000, dst_reg, src_reg); + /* ag %dst,percpu_offset */ + EMIT6_DISP_LH(0xe3000000, 0x0008, dst_reg, REG_0, REG_0, + (unsigned long)percpu_offset); + break; + } switch (insn->off) { case 0: /* DST = SRC */ /* lgr %dst,%src */ @@ -1866,9 +1877,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, if ((void *)func == arch_bpf_timed_may_goto) { /* * arch_bpf_timed_may_goto() has a special ABI: the - * parameters are in BPF_REG_AX and BPF_REG_10; the - * return value is in BPF_REG_AX; and all GPRs except - * REG_W0, REG_W1, and BPF_REG_AX are callee-saved. + * parameter is in BPF_REG_AX; the return value is in + * BPF_REG_AX; and all GPRs except REG_W0, REG_W1, and + * BPF_REG_AX are callee-saved. */ /* brasl %r0,func */ @@ -3114,6 +3125,11 @@ bool bpf_jit_supports_timed_may_goto(void) return true; } +bool bpf_jit_supports_percpu_insn(void) +{ + return true; +} + bool bpf_jit_inlines_helper_call(s32 imm) { switch (imm) { diff --git a/arch/s390/net/bpf_timed_may_goto.S b/arch/s390/net/bpf_timed_may_goto.S index 06f567a460d7..05f532f8c639 100644 --- a/arch/s390/net/bpf_timed_may_goto.S +++ b/arch/s390/net/bpf_timed_may_goto.S @@ -21,9 +21,9 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* - * This function has a special ABI: the parameters are in %r12 and - * %r13; the return value is in %r12; all GPRs except %r0, %r1, and - * %r12 are callee-saved; and the return address is in %r0. + * This function has a special ABI: the parameter is in %r12; the + * return value is in %r12; all GPRs except %r0, %r1, and %r12 are + * callee-saved; and the return address is in %r0. */ stmg %r2,%r5,FRAME_OFF+R2_OFF(%r15) stg %r14,FRAME_OFF+R14_OFF(%r15) @@ -33,7 +33,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) lay %r15,-FRAME_SIZE(%r15) stg %r1,__SF_BACKCHAIN(%r15) - lay %r2,0(%r12,%r13) + lgr %r2,%r12 brasl %r14,bpf_check_timed_may_goto lgr %r12,%r2 diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c9419bc65f08..4c3e09d98131 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3441,18 +3441,6 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int * ip += emit_kfunc_arg_moves(fm, outgoing_arg_base - outgoing_rsp, &prog); } - if (func == (u8 *)arch_bpf_timed_may_goto) { - u32 fp = priv_frame_ptr ? X86_REG_R9 : BPF_REG_FP; - - /* - * AX has the offset of count and timestamp - * in the stack. Turn it into a pointer. - * add r10, rbp or add r10, r9 - */ - maybe_emit_mod(&prog, BPF_REG_AX, fp, true); - EMIT2(0x01, add_2reg(0xC0, BPF_REG_AX, fp)); - ip += 3; - } if (priv_frame_ptr) { push_r9(&prog); ip += 2; diff --git a/arch/x86/net/bpf_timed_may_goto.S b/arch/x86/net/bpf_timed_may_goto.S index dd2dde89836b..da42de7cca98 100644 --- a/arch/x86/net/bpf_timed_may_goto.S +++ b/arch/x86/net/bpf_timed_may_goto.S @@ -25,10 +25,7 @@ SYM_FUNC_START(arch_bpf_timed_may_goto) /* Save stack argument 6. */ pushq %r9 - /* - * r10 has the pointer to count and timestamp. JIT added the frame - * pointer to the offset. Pass it as first argument. - */ + /* r10 has the pointer to count and timestamp. Pass it as first argument. */ movq %r10, %rdi /* Emit call depth accounting for call below. */ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0b44bcb4f56e..fdbf638300fc 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1926,6 +1926,8 @@ struct bpf_prog_aux { const char *attach_func_name; struct bpf_prog **func; struct bpf_prog_aux *main_prog_aux; + /* main prog: there when the prog has timed may_goto */ + struct bpf_timed_may_goto __percpu *timed_may_goto; void *jit_data; /* JIT specific data. arch dependent */ struct bpf_jit_poke_descriptor *poke_tab; struct bpf_kfunc_desc_tab *kfunc_tab; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 228a3f6889c2..b43eb97bcd44 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -969,6 +969,10 @@ struct bpf_subprog_info { bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; + /* bpf_loop() or bpf_for_each_map_elem() call it in a loop */ + bool is_loop_cb: 1; + /* BPF code, another helper or a kfunc call it */ + bool is_called: 1; bool args_cached: 1; /* true if the return value is passed in the R0:R2 register pair */ bool ret_reg_pair: 1; diff --git a/include/linux/filter.h b/include/linux/filter.h index 98254742ae37..4f0cd67caccb 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -758,9 +758,15 @@ struct bpf_prog_stats { struct u64_stats_sync syncp; } __aligned(2 * sizeof(u64)); +/* + * Per-cpu memory of a prog. All may_goto of one invocation of the prog use it. An invocation is + * one call from the kernel into the main prog, with the subprogs and sync callbacks it reaches. + */ struct bpf_timed_may_goto { u64 count; u64 timestamp; + /* the async callback that has set count and timestamp, see bpf_timed_may_goto_save() */ + void *owner; }; struct sk_filter { @@ -1328,6 +1334,8 @@ u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *); +void bpf_timed_may_goto_save(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved); +void bpf_timed_may_goto_restore(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved); bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id); static inline bool bpf_dump_raw_ok(const struct cred *cred) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index b9709e3b7656..831f73f82599 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -294,6 +294,7 @@ void __bpf_prog_free(struct bpf_prog *fp) mutex_destroy(&fp->aux->used_maps_mutex); mutex_destroy(&fp->aux->dst_mutex); kfree(fp->aux->poke_tab); + free_percpu(fp->aux->timed_may_goto); kfree(fp->aux); } free_percpu(fp->stats); @@ -3645,6 +3646,38 @@ u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p) return BPF_MAX_TIMED_LOOPS; } +/* + * The main prog sets count and timestamp in its first insns. A call of an async callback is an + * invocation of its own too, but it can interrupt an invocation of the prog on this cpu, or go on + * next to one that is preempted. So its caller sets count and timestamp, and puts the old ones + * back at the end. Not when another invocation has set them in the meantime: they are for that + * one now. + * The caller has disabled migration. + */ +void bpf_timed_may_goto_save(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved) +{ + struct bpf_timed_may_goto *cur; + + if (!prog->aux->timed_may_goto) + return; + cur = this_cpu_ptr(prog->aux->timed_may_goto); + *saved = *cur; + cur->count = BPF_MAX_TIMED_LOOPS; + cur->timestamp = 0; + cur->owner = saved; +} + +void bpf_timed_may_goto_restore(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved) +{ + struct bpf_timed_may_goto *cur; + + if (!prog->aux->timed_may_goto) + return; + cur = this_cpu_ptr(prog->aux->timed_may_goto); + if (cur->owner == saved) + *cur = *saved; +} + /* for configs without MMU or 32-bit */ __weak const struct bpf_map_ops arena_map_ops; __weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 41bc37e205e0..a656634fa9e4 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1803,6 +1803,50 @@ static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off, return cnt + tail_cnt; } +/* + * Timed may_goto. All may_goto of a prog use one count and one timestamp: per-cpu memory of the + * prog, set in the first insns of the main prog, and by the callers of async callbacks. + * @reg holds the address. It is a register that is dead at the may_goto, or R0 that is saved at + * @save_off in the stack. + */ +static int may_goto_expand_timed(struct bpf_insn *insn_buf, int off, + struct bpf_timed_may_goto __percpu *tmg, int reg, int save_off) +{ + struct bpf_insn ld_tmg[] = { BPF_LD_IMM64(reg, (__force unsigned long)tmg) }; + int cnt = 0, jmp, tail_cnt = save_off ? 6 : 5; + + if (save_off) + insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, reg, save_off); + insn_buf[cnt++] = ld_tmg[0]; + insn_buf[cnt++] = ld_tmg[1]; + insn_buf[cnt++] = BPF_MOV64_PERCPU_REG(reg, reg); + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, reg, 0); + if (save_off) { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_10, save_off); + jmp = cnt++; + insn_buf[jmp] = BPF_JMP32_A(off >= 0 ? off + tail_cnt : off - jmp); + } else { + jmp = cnt; + off = off >= 0 ? off + tail_cnt : off - jmp; + if (off == (s16)off) { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off); + } else { + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1); + insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1); + } + } + insn_buf[cnt++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1); + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2); + /* AX is the argument and the return value */ + insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_AX, reg); + insn_buf[cnt++] = BPF_EMIT_CALL(arch_bpf_timed_may_goto); + insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, reg, BPF_REG_AX, 0); + if (save_off) + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_10, save_off); + return cnt; +} + /* Do various post-verification rewrites in a single program pass. * These rewrites simplify JIT and interpreter implementations. */ @@ -1823,6 +1867,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) struct bpf_subprog_info *subprogs = env->subprog_info; u16 stack_depth = subprogs[cur_subprog].stack_depth; u16 stack_depth_extra = 0; + /* arch_bpf_timed_may_goto() takes and returns AX in a cpu register: for JITs only */ + bool timed_may_goto = prog->jit_requested && bpf_jit_supports_timed_may_goto(); + struct bpf_timed_may_goto __percpu *tmg = NULL; if (env->seen_exception && !env->exception_callback_subprog) { struct bpf_insn *patch = insn_buf; @@ -2078,38 +2125,28 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } - if (bpf_is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) { - int stack_off_cnt = -stack_depth - 16; - /* - * AX is used as an argument to pass in stack_off_cnt - * (to add to r10/fp), and also as the return value of - * the call to arch_bpf_timed_may_goto. - */ - struct bpf_insn tail[] = { - BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1), - BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2), - BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt), - BPF_EMIT_CALL(arch_bpf_timed_may_goto), - BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt), - }; + if (bpf_is_may_goto_insn(insn) && timed_may_goto) { + u16 live = env->insn_aux_data[i + delta].live_regs_before; + int reg, save_off = 0; - /* - * Two 8 byte slots, depth-16 stores the count, and - * depth-8 stores the start timestamp of the loop. - * - * The starting value of count is BPF_MAX_TIMED_LOOPS - * (0xffff). Every iteration loads it and subs it by 1, - * until the value becomes 0 in AX (thus, 1 in stack), - * after which we call arch_bpf_timed_may_goto, which - * either sets AX to 0xffff to keep looping, or to 0 - * upon timeout. AX is then stored into the stack. In - * the next iteration, we either see 0 and break out, or - * continue iterating until the next time value is 0 - * after subtraction, rinse and repeat. - */ - stack_depth_extra = 16; - cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, - tail, ARRAY_SIZE(tail)); + /* a register that is dead here holds the address of the count */ + for (reg = BPF_REG_0; reg <= BPF_REG_9; reg++) + if (!(live & BIT(reg))) + break; + if (reg > BPF_REG_9) { + reg = BPF_REG_0; + save_off = -stack_depth - 8; + stack_depth_extra = 8; + } + if (!tmg) { + tmg = alloc_percpu_gfp(struct bpf_timed_may_goto, + GFP_KERNEL_ACCOUNT); + if (!tmg) + return -ENOMEM; + prog->aux->timed_may_goto = tmg; + } + prog->jit_required = 1; + cnt = may_goto_expand_timed(insn_buf, insn->off, tmg, reg, save_off); new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); if (!new_prog) @@ -2730,24 +2767,52 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) env->prog->aux->stack_depth = subprogs[0].stack_depth; for (i = 0; i < env->subprog_cnt; i++) { - int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; int subprog_start = subprogs[i].start; int stack_slots = subprogs[i].stack_extra / 8; - int slots = delta, cnt = 0; + /* an invocation begins, see bpf_timed_may_goto_save() for async callbacks */ + bool main_prog = tmg && !i; + /* only a helper calls it, in a loop that ends when it returns 1 */ + bool loop_cb = tmg && i && subprogs[i].is_loop_cb && !subprogs[i].is_called && + !subprogs[i].is_async_cb; + int cnt = 0; - if (!stack_slots) + if (timed_may_goto ? !main_prog && !loop_cb : !stack_slots) continue; - /* We need two slots in case timed may_goto is supported. */ - if (stack_slots > slots) { + if (stack_slots > 1) { verifier_bug(env, "stack_slots supports may_goto only"); return -EFAULT; } stack_depth = subprogs[i].stack_depth; - if (bpf_jit_supports_timed_may_goto()) { - insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, - BPF_MAX_TIMED_LOOPS); - insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0); + if (timed_may_goto) { + struct bpf_insn ld_tmg[] = { + BPF_LD_IMM64(BPF_REG_0, (__force unsigned long)tmg) + }; + int count_off = offsetof(struct bpf_timed_may_goto, count); + int timestamp_off = offsetof(struct bpf_timed_may_goto, timestamp); + int owner_off = offsetof(struct bpf_timed_may_goto, owner); + + /* R0 has nothing at the entry */ + insn_buf[cnt++] = ld_tmg[0]; + insn_buf[cnt++] = ld_tmg[1]; + insn_buf[cnt++] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); + if (main_prog) { + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, count_off, + BPF_MAX_TIMED_LOOPS); + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, timestamp_off, 0); + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, owner_off, 0); + } else { + /* + * The time is up, but the helper would call the function again and + * again. End its loop. The verifier has seen that: the loop can end + * after each call. + */ + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, + count_off); + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2); + insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 1); + insn_buf[cnt++] = BPF_EXIT_INSN(); + } } else { /* Add ST insn to subprog prologue to init extra stack */ insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, @@ -2774,7 +2839,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) * to insn after BPF_ST that inits may_goto count. * Adjustment will succeed because bpf_patch_insn_data() didn't fail. */ - WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta)); + WARN_ON(adjust_jmp_off(env->prog, subprog_start, cnt - 1)); } /* Since poke tab is now finalized, publish aux to tracker. */ diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 22b46f7d0043..6788489a2608 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -1222,6 +1222,8 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) { struct bpf_hrtimer *t = container_of(hrtimer, struct bpf_hrtimer, timer); struct bpf_map *map = READ_ONCE(t->cb.map); + struct bpf_prog *prog = READ_ONCE(t->cb.prog); + struct bpf_timed_may_goto saved; void *value = t->cb.value; bpf_callback_t callback_fn; void *key; @@ -1229,7 +1231,7 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) BTF_TYPE_EMIT(struct bpf_timer); callback_fn = rcu_dereference_check(t->cb.callback_fn, rcu_read_lock_bh_held()); - if (!callback_fn || !map) + if (!callback_fn || !map || !prog) goto out; /* bpf_timer_cb() runs in hrtimer_run_softirq. It doesn't migrate and @@ -1242,8 +1244,10 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer) key = map_key_from_value(map, value, &idx); + bpf_timed_may_goto_save(prog, &saved); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); /* The verifier checked that return value is zero. */ + bpf_timed_may_goto_restore(prog, &saved); this_cpu_write(hrtimer_running, NULL); out: @@ -1254,6 +1258,8 @@ static void bpf_wq_work(struct work_struct *work) { struct bpf_work *w = container_of(work, struct bpf_work, work); struct bpf_async_cb *cb = &w->cb; + struct bpf_timed_may_goto saved; + struct bpf_prog *prog; struct bpf_map *map; bpf_callback_t callback_fn; void *value = cb->value; @@ -1267,14 +1273,17 @@ static void bpf_wq_work(struct work_struct *work) map = READ_ONCE(cb->map); callback_fn = READ_ONCE(cb->callback_fn); - if (!callback_fn || !map) + prog = READ_ONCE(cb->prog); + if (!callback_fn || !map || !prog) goto out; key = map_key_from_value(map, value, &idx); migrate_disable(); + bpf_timed_may_goto_save(prog, &saved); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + bpf_timed_may_goto_restore(prog, &saved); migrate_enable(); out: @@ -4611,6 +4620,7 @@ static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx, static void bpf_task_work_callback(struct callback_head *cb) { struct bpf_task_work_ctx *ctx = container_of(cb, struct bpf_task_work_ctx, work); + struct bpf_timed_may_goto saved; enum bpf_task_work_state state; u32 idx; void *key; @@ -4632,8 +4642,10 @@ static void bpf_task_work_callback(struct callback_head *cb) key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx); migrate_disable(); + bpf_timed_may_goto_save(ctx->prog, &saved); ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key, (u64)(long)ctx->map_val, 0, 0); + bpf_timed_may_goto_restore(ctx->prog, &saved); migrate_enable(); bpf_task_work_ctx_reset(ctx); @@ -4873,6 +4885,7 @@ static void bpf_rcu_run_callback(struct rcu_head *rcu) struct bpf_rcu_head_kern *rh = container_of(rcu, struct bpf_rcu_head_kern, rcu); bpf_callback_t callback_fn = rh->callback_fn; struct bpf_prog *prog = rh->prog; + struct bpf_timed_may_goto saved; struct bpf_map *map = rh->map; void *value, *key; u32 idx; @@ -4884,7 +4897,9 @@ static void bpf_rcu_run_callback(struct rcu_head *rcu) smp_store_release(&rh->armed, 0); rcu_read_lock_dont_migrate(); + bpf_timed_may_goto_save(prog, &saved); callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0); + bpf_timed_may_goto_restore(prog, &saved); rcu_read_unlock_migrate(); bpf_prog_put(prog); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fb96c61ebec0..5e4700fda66b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11508,6 +11508,12 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins if (err) return err; + if (!bpf_pseudo_kfunc_call(insn) && + (insn->imm == BPF_FUNC_loop || insn->imm == BPF_FUNC_for_each_map_elem)) + env->subprog_info[subprog].is_loop_cb = true; + else + env->subprog_info[subprog].is_called = true; + /* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ @@ -11544,6 +11550,7 @@ static int check_static_func_call(struct bpf_verifier_env *env, int subprog, int err; caller = state->frame[state->curframe]; + env->subprog_info[subprog].is_called = true; /* * Track caller's total stack arg count (incoming + max outgoing). diff --git a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c index ea1fac95b461..017add229bef 100644 --- a/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c +++ b/tools/testing/selftests/bpf/libarena/selftests/test_parallel_bitmap.bpf.c @@ -11,6 +11,7 @@ #define TEST_BITMAP_BITS (2 * BITS_PER_LONG_LONG) #define TEST_BITMAP_SYNC_SPINS BPF_MAX_LOOPS #define TEST_BITMAP_ITERS 10 * 1000 * 1000 +#define TEST_BITMAP_NSEC (50 * 1000 * 1000) static struct arena_bitmap __arena *bitmap; static volatile u64 started; @@ -129,6 +130,7 @@ static int bitmap_test_common_single(int parity) static int bitmap_test_common(int parity) { + u64 start; int ret; u32 i; @@ -138,6 +140,7 @@ static int bitmap_test_common(int parity) if (ret) return ret; + start = bpf_ktime_get_ns(); for (i = zero; i < TEST_BITMAP_ITERS && can_loop; i++) { ret = bitmap_test_common_single(parity); if (ret) @@ -147,6 +150,13 @@ static int bitmap_test_common(int parity) break; bitmap_test_reset_single(parity); + + /* + * Stop before the time slice of may_goto is used up. After that the + * bmp_*() functions can't retry a cmpxchg that fails. + */ + if (bpf_ktime_get_ns() - start > TEST_BITMAP_NSEC) + break; } return 0; diff --git a/tools/testing/selftests/bpf/prog_tests/may_goto_far.c b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c index 8d0ff30bd068..b6d262911a3e 100644 --- a/tools/testing/selftests/bpf/prog_tests/may_goto_far.c +++ b/tools/testing/selftests/bpf/prog_tests/may_goto_far.c @@ -34,8 +34,10 @@ static struct bpf_insn *fill(struct bpf_insn *insn, int cnt, int reg) * The prog returns 0 when may_goto jumps to its target and 2 when * it lands in one of 'r0 = r8' areas. There are no constants between * may_goto and its target to keep the distance when constants are blinded. + * + * With 'subprog' all of that is a subprog. */ -static void test_far(int off) +static void test_far(int off, bool subprog) { LIBBPF_OPTS(bpf_prog_load_opts, opts, .prog_flags = BPF_F_SLEEPABLE); LIBBPF_OPTS(bpf_test_run_opts, topts); @@ -50,6 +52,10 @@ static void test_far(int off) return; insn = insns; + if (subprog) { + *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_CALL, 0, 1); + *insn++ = BPF_EXIT_INSN(); + } *insn++ = BPF_EMIT_CALL(BPF_FUNC_ktime_get_ns); *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_0); *insn++ = BPF_ALU64_IMM(BPF_RSH, BPF_REG_6, 63); @@ -82,11 +88,19 @@ static void test_far(int off) void test_may_goto_far(void) { if (test__start_subtest("32762")) - test_far(32762); + test_far(32762, false); if (test__start_subtest("32767")) - test_far(32767); + test_far(32767, false); if (test__start_subtest("-32767")) - test_far(-32767); + test_far(-32767, false); if (test__start_subtest("-32768")) - test_far(-32768); + test_far(-32768, false); + if (test__start_subtest("subprog 1")) + test_far(1, true); + if (test__start_subtest("subprog 32767")) + test_far(32767, true); + if (test__start_subtest("subprog -6")) + test_far(-6, true); + if (test__start_subtest("subprog -32768")) + test_far(-32768, true); } diff --git a/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c b/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c new file mode 100644 index 000000000000..3da3f33a2898 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/timed_may_goto.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include +#include +#include +#include "timed_may_goto.skel.h" +#include "timed_may_goto_user.skel.h" +#include "timed_may_goto_ext.skel.h" + +#define TRIES 4 + +static int run_tc(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .repeat = 1, + ); + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &opts), "test_run")) + return -1; + return opts.retval; +} + +static int run_syscall(struct bpf_program *prog) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(prog), &opts), "test_run")) + return -1; + return opts.retval; +} + +static void wait_for(int *val, int expected, const char *name) +{ + int i; + + for (i = 0; i < 500 && *(volatile int *)val != expected; i++) + usleep(10000); + ASSERT_EQ(*val, expected, name); +} + +static void test_async(struct timed_may_goto_user *skel) +{ + ASSERT_EQ(run_syscall(skel->progs.async_after_timeout), 0, "async_after_timeout"); + wait_for(&skel->bss->timer_cnt, 1000, "timer_cnt"); + wait_for(&skel->bss->wq_cnt, 1000, "wq_cnt"); + wait_for(&skel->bss->tw_cnt, 1000, "tw_cnt"); + wait_for(&skel->bss->rcu_cnt, 1000, "rcu_cnt"); + wait_for(&skel->bss->rcu_trace_cnt, 1000, "rcu_trace_cnt"); +} + +static void test_timer_on_top(struct timed_may_goto_user *skel) +{ + int ms = run_syscall(skel->progs.spin_under_timer); + int ticks = skel->bss->ticks; + + /* stop the timer */ + skel->bss->ticks = 100; + ASSERT_GE(ms, 200, "ms"); + ASSERT_LE(ms, 500, "ms"); + /* control: the callback interrupted the prog, and its loop went on to the end */ + ASSERT_GE(ticks, 5, "ticks"); + ASSERT_LT(ticks, 100, "ticks"); +} + +static void test_freplace(struct timed_may_goto_user *skel) +{ + struct timed_may_goto_ext *ext; + + ASSERT_EQ(run_tc(skel->progs.freplace_target), 1, "not replaced"); + + ext = timed_may_goto_ext__open(); + if (!ASSERT_OK_PTR(ext, "ext_open")) + return; + bpf_program__set_attach_target(ext->progs.replacement, + bpf_program__fd(skel->progs.freplace_target), + "replaced_global"); + if (!ASSERT_OK(timed_may_goto_ext__load(ext), "ext_load") || + !ASSERT_OK(timed_may_goto_ext__attach(ext), "ext_attach")) + goto out; + ASSERT_EQ(run_tc(skel->progs.freplace_target), TRIES, "replaced"); +out: + timed_may_goto_ext__destroy(ext); + ASSERT_EQ(run_tc(skel->progs.freplace_target), 1, "not replaced"); +} + +/* What a prog leaves behind when its time is up doesn't matter for the next one on the cpu. */ +static void test_tail_call_and_throw(struct timed_may_goto_user *skel) +{ + int key = 0, fd = bpf_program__fd(skel->progs.tail_callee); + int map_fd = bpf_map__fd(skel->maps.jmp_table); + + if (!ASSERT_OK(bpf_map_update_elem(map_fd, &key, &fd, 0), "update")) + return; + + ASSERT_EQ(run_tc(skel->progs.normal_prog), 1, "normal_prog"); + ASSERT_EQ(run_tc(skel->progs.normal_prog), 1, "normal_prog"); + + ASSERT_EQ(run_tc(skel->progs.tail_call_in_subprog), 1, "tail_call_in_subprog"); + ASSERT_EQ(skel->bss->tail_callee_spun, 1, "tail_callee_spun"); + + ASSERT_EQ(run_tc(skel->progs.tail_call_after_timeout), 0, "tail_call_after_timeout"); + ASSERT_EQ(skel->bss->tail_callee_spun, 1, "tail_callee_spun"); + + ASSERT_EQ(run_tc(skel->progs.throw_after_timeout), 7, "throw_after_timeout"); + ASSERT_EQ(run_tc(skel->progs.throw_after_timeout), 7, "throw_after_timeout"); +} + +static void test_user_ringbuf(struct timed_may_goto_user *skel) +{ + struct user_ring_buffer *rb; + void *sample; + int i; + + rb = user_ring_buffer__new(bpf_map__fd(skel->maps.user_ringbuf), NULL); + if (!ASSERT_OK_PTR(rb, "user_ring_buffer__new")) + return; + for (i = 0; i < TRIES; i++) { + sample = user_ring_buffer__reserve(rb, 8); + if (!ASSERT_OK_PTR(sample, "reserve")) + goto out; + user_ring_buffer__submit(rb, sample); + } + /* The helper takes a sample out of the buffer for each call: all calls have to run. */ + ASSERT_EQ(run_syscall(skel->progs.drain), TRIES * 10 + 1, "drain"); +out: + user_ring_buffer__free(rb); +} + +static void test_priv_stack(struct timed_may_goto_user *skel) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + int fd = bpf_program__fd(skel->progs.priv_stack); + struct bpf_link *link; + + link = bpf_program__attach(skel->progs.priv_stack); + if (!ASSERT_OK_PTR(link, "attach")) + return; + ASSERT_OK(bpf_prog_test_run_opts(fd, &opts), "test_run"); + ASSERT_EQ(skel->bss->priv_stack_spun, 1, "priv_stack_spun"); + ASSERT_EQ(skel->bss->priv_stack_cb_spun, 0, "priv_stack_cb_spun"); + bpf_link__destroy(link); +} + +/* + * Serial: most progs spin for 250 ms in an RCU read-side critical section, one after the other. + * That delays what other tests wait for. In parallel mode "mmap" gives up its wait for the end of + * a map. + */ +void serial_test_timed_may_goto(void) +{ + struct timed_may_goto_user *skel; + cpu_set_t old, new; + + /* The interpreter counts in each frame. */ + if (!is_jit_enabled()) { + test__skip(); + return; + } + + RUN_TESTS(timed_may_goto); + +#if !defined(__x86_64__) && !defined(__aarch64__) + return; +#endif + skel = timed_may_goto_user__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + /* Count and timestamp are per-cpu: all progs of a subtest run on one cpu. */ + if (!ASSERT_OK(sched_getaffinity(0, sizeof(old), &old), "getaffinity")) + goto out; + CPU_ZERO(&new); + CPU_SET(sched_getcpu(), &new); + if (!ASSERT_OK(sched_setaffinity(0, sizeof(new), &new), "setaffinity")) + goto out; + + if (test__start_subtest("async")) + test_async(skel); + if (test__start_subtest("timer_on_top")) + test_timer_on_top(skel); + if (test__start_subtest("freplace")) + test_freplace(skel); + if (test__start_subtest("tail_call_and_throw")) + test_tail_call_and_throw(skel); + if (test__start_subtest("user_ringbuf")) + test_user_ringbuf(skel); + if (test__start_subtest("priv_stack")) + test_priv_stack(skel); + + sched_setaffinity(0, sizeof(old), &old); +out: + timed_may_goto_user__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/arena_htab.c b/tools/testing/selftests/bpf/progs/arena_htab.c index 81eaa94afeb0..80cc14bb1c0f 100644 --- a/tools/testing/selftests/bpf/progs/arena_htab.c +++ b/tools/testing/selftests/bpf/progs/arena_htab.c @@ -34,8 +34,6 @@ int arena_htab_llvm(void *ctx) cast_kern(htab); htab_init(htab); - cast_kern(arr); - /* first run. No old elems in the table */ for (i = zero; i < 100000 && can_loop; i++) { htab_update_elem(htab, i, i); diff --git a/tools/testing/selftests/bpf/progs/timed_may_goto.c b/tools/testing/selftests/bpf/progs/timed_may_goto.c new file mode 100644 index 000000000000..c1c48af808a9 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/timed_may_goto.c @@ -0,0 +1,898 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * There is one time slice of may_goto for an invocation of a prog: one call from the kernel into + * the main prog, with the subprogs and sync callbacks that it reaches. + * + * Most tests call a function that spins until may_goto stops it, four times, each in a different + * way. Only the first call can spin. With a time slice for each call all four do. + */ +#include +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +char _license[] SEC("license") = "GPL"; + +#if (defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)) && __clang_major__ >= 18 + +#define TRIES 4 + +int zero; + +/* Return 1 if the loop went on until the time was up. */ +static __always_inline int spin(void) +{ + __u64 t0 = bpf_ktime_get_ns(); + + while (can_loop) + ; + return bpf_ktime_get_ns() - t0 > 100 * 1000 * 1000; +} + +static __noinline int spins(void) +{ + return spin(); +} + +static __noinline int counts_to_1000(void) +{ + int i, cnt = zero; + + for (i = zero; i < 1000 && can_loop; i++) + cnt++; + return cnt; +} + +SEC("tc") +__description("timed may_goto: call") +__success __retval(1) +int call_subprog(void *ctx) +{ + int i, n = 0; + + for (i = 0; i < TRIES; i++) + n += spins(); + return n; +} + +SEC("tc") +__description("timed may_goto: a call after the time is up makes no iteration") +__success __retval(0) +int short_loop_after_timeout(void *ctx) +{ + spins(); + return counts_to_1000(); +} + +SEC("tc") +__description("timed may_goto: the loop of the main prog stops when the time is up in a subprog") +__success __retval(1) +int main_loop_after_timeout(void *ctx) +{ + int n = 0; + + while (can_loop && n < 2 * TRIES) + n += spins(); + return n; +} + +static __noinline int passes(int i) +{ + return spins() + i; +} + +SEC("tc") +__description("timed may_goto: call through a function without a loop") +__success __retval(1) +int call_through(void *ctx) +{ + int i, n = 0; + + for (i = 0; i < TRIES; i++) + n += passes(zero); + return n; +} + +__noinline int global_spins(void) +{ + return spin(); +} + +SEC("tc") +__description("timed may_goto: call of a global function") +__success __retval(1) +int call_global(void *ctx) +{ + int i, n = 0; + + for (i = 0; i < TRIES; i++) + n += global_spins(); + return n; +} + +SEC("tc") +__description("timed may_goto: callx") +__success __retval(1) +__naked void callx_subprog(void) +{ + asm volatile ( + "r6 = 0;" + "r7 = 0;" + "1:" + "r3 = %[spins] ll;" + "callx r3;" + "r6 += r0;" + "r7 += 1;" + "if r7 < 4 goto 1b;" + "r0 = r6;" + "exit;" + : + : __imm_addr(spins) + : __clobber_all); +} + +/* With a time slice for each call it gives up, so that the test fails and doesn't lock up. */ +static int loop_cb(__u64 i, int *spun) +{ + *spun += spin(); + return *spun > 2 * TRIES; +} + +SEC("tc") +__description("timed may_goto: bpf_loop() inlined") +__success __retval(1) +int loop_inlined(void *ctx) +{ + int spun = 0; + + bpf_loop(1 << 20, loop_cb, &spun, 0); + return spun; +} + +SEC("tc") +__description("timed may_goto: bpf_loop() not inlined") +__success __retval(1) +int loop_not_inlined(void *ctx) +{ + int spun = 0; + + bpf_loop(1 << 20, loop_cb, &spun, zero); + return spun; +} + +static int count_cb(__u64 i, int *calls) +{ + (*calls)++; + return 0; +} + +/* The time is up: the helper calls the callback one more time. It returns 1 in its first insns. */ +#define LOOP_ENDS(name, flags) \ +SEC("tc") \ +__description("timed may_goto: bpf_loop() " #name " ends when the time is up") \ +__success __retval(10) \ +int loop_ends_##name(void *ctx) \ +{ \ + int calls = 0; \ + \ + spins(); \ + return bpf_loop(1000, count_cb, &calls, flags) * 10 + calls; \ +} + +LOOP_ENDS(inlined, 0) +LOOP_ENDS(not_inlined, zero) + +struct nested { + __u64 t0; + int spun; +}; + +/* Without an end of bpf_loop(): give up, so that the test fails and doesn't lock up. */ +static int inner_cb(__u64 i, struct nested *n) +{ + if (!n->spun) + n->spun = spin(); + return (i & 1023) == 0 && bpf_ktime_get_ns() - n->t0 > 1000 * 1000 * 1000; +} + +static int outer_cb(__u64 i, struct nested *n) +{ + bpf_loop(1 << 23, inner_cb, n, 0); + return bpf_ktime_get_ns() - n->t0 > 1000 * 1000 * 1000; +} + +SEC("tc") +__description("timed may_goto: bpf_loop() in bpf_loop() ends when the time is up") +__success __retval(1) +int loop_in_loop(void *ctx) +{ + struct nested n = { .t0 = bpf_ktime_get_ns() }; + + bpf_loop(1 << 23, outer_cb, &n, 0); + return n.spun && bpf_ktime_get_ns() - n.t0 < 500 * 1000 * 1000; +} + +static __noinline int called_cb(__u64 i, int *calls) +{ + (*calls)++; + return 0; +} + +/* BPF code calls the callback too, so its return value has to be its own. bpf_loop() goes on. */ +SEC("tc") +__description("timed may_goto: bpf_loop() with a callback that is called as well") +__success __retval(45) +int loop_and_call(void *ctx) +{ + int calls = 0; + + spins(); + if (called_cb(zero, &calls)) + return 100; + return bpf_loop(TRIES, called_cb, &calls, 0) * 10 + calls; +} + +static int short_cb(__u64 i, __u64 *t0) +{ + int j; + + for (j = zero; j < 60000 && can_loop; j++) + ; + return (i & 1023) == 0 && bpf_ktime_get_ns() - *t0 > 1000 * 1000 * 1000; +} + +/* + * The main prog has no may_goto. Each call of the callback ends before it reads the clock. The + * iterations of all calls add up. + */ +SEC("tc") +__description("timed may_goto: bpf_loop() the callback makes 60000 iterations") +__success __retval(1) +int loop_short_cb(void *ctx) +{ + __u64 t0 = bpf_ktime_get_ns(); + + bpf_loop(1 << 20, short_cb, &t0, 0); + return bpf_ktime_get_ns() - t0 < 1000 * 1000 * 1000; +} + +/* spins() uses up the time. The loop that calls it ends at its next may_goto. */ +static __noinline int loops_around_spins(void) +{ + int n = zero; + + while (can_loop && n < 20000) { + if (n < 2 * TRIES) + spins(); + n++; + } + return n; +} + +SEC("tc") +__description("timed may_goto: a loop that is on the stack ends when the time is up") +__success __retval(1) +int rest_of_batch(void *ctx) +{ + return loops_around_spins(); +} + +static __noinline int one_iteration(void) +{ + int i, n = zero; + + for (i = zero; i < 1 && can_loop; i++) + n++; + return n; +} + +long calls_ok; + +static int call_one(__u64 i, __u64 *t0) +{ + if (!one_iteration()) + return 1; + calls_ok++; + /* With a time slice for each call: give up. */ + return (i & 1023) == 0 && bpf_ktime_get_ns() - *t0 > 1000 * 1000 * 1000; +} + +static int call_many(__u64 i, __u64 *t0) +{ + long before = calls_ok; + + bpf_loop(1 << 23, call_one, t0, 0); + return calls_ok - before != 1 << 23; +} + +/* + * The iterations of all calls count: the prog reads the clock in each 65535th call. The call that + * finds out that the time is up makes its iteration. + */ +SEC("tc") +__description("timed may_goto: 65535 calls that make one iteration read the clock") +__success __retval(1) +int clock_after_65535_calls(void *ctx) +{ + __u64 t0 = bpf_ktime_get_ns(); + + calls_ok = 0; + bpf_loop(1000, call_many, &t0, 0); + return calls_ok > 0 && calls_ok % 65535 == 0; +} + +#define MAP(name, map_type, flags) \ +struct { \ + __uint(type, map_type); \ + __uint(max_entries, TRIES); \ + __uint(map_flags, flags); \ + __type(key, int); \ + __type(value, int); \ +} name SEC(".maps") + +MAP(array, BPF_MAP_TYPE_ARRAY, 0); +MAP(hash, BPF_MAP_TYPE_HASH, 0); + +/* The first call uses up the time. The second call ends the loop of the helper. */ +static int elem_cb(void *map, int *key, int *val, int *spun) +{ + *spun += spin(); + return 0; +} + +#define FOR_EACH(name) \ +SEC("tc") \ +__description("timed may_goto: bpf_for_each_map_elem() " #name) \ +__success __retval(21) \ +int for_each_##name(void *ctx) \ +{ \ + int i, key, spun = 0; \ + \ + for (i = 0; i < TRIES; i++) { \ + key = i; \ + bpf_map_update_elem(&name, &key, &key, 0); \ + } \ + return bpf_for_each_map_elem(&name, elem_cb, &spun, 0) * 10 + spun; \ +} + +FOR_EACH(array) +FOR_EACH(hash) + +static long vma_cb(struct task_struct *task, struct vm_area_struct *vma, int *spun) +{ + *spun += spin(); + return 0; +} + +SEC("raw_tp") +__description("timed may_goto: bpf_find_vma()") +__success __retval(1) +int find_vma(void *ctx) +{ + struct task_struct *task = bpf_get_current_task_btf(); + __u64 addr = task->mm->arg_start; + int i, spun = 0; + + for (i = 0; i < TRIES; i++) + if (bpf_find_vma(task, addr, vma_cb, &spun, 0)) + return 100; + return spun; +} + +struct tree_node { + struct bpf_rb_node rb; + int key; +}; + +#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8))) +private(A) struct bpf_spin_lock lock; +private(A) struct bpf_rb_root root __contains(tree_node, rb); + +int less_spun, less_calls; + +static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + long n = zero; + + /* no helpers under the lock */ + while (can_loop) + n++; + less_calls++; + less_spun += n > 1000 * 1000; + return container_of(a, struct tree_node, rb)->key < + container_of(b, struct tree_node, rb)->key; +} + +SEC("tc") +__description("timed may_goto: bpf_rbtree_add()") +__success __retval(1) +int rbtree_add(void *ctx) +{ + struct bpf_rb_node *rb; + struct tree_node *n; + int i; + + less_spun = less_calls = 0; + for (i = 0; i < TRIES; i++) { + n = bpf_obj_new(typeof(*n)); + if (!n) + return 100; + n->key = TRIES - i; + bpf_spin_lock(&lock); + bpf_rbtree_add(&root, &n->rb, less); + bpf_spin_unlock(&lock); + } + for (i = 0; i < TRIES; i++) { + bpf_spin_lock(&lock); + rb = bpf_rbtree_first(&root); + if (rb) + rb = bpf_rbtree_remove(&root, rb); + bpf_spin_unlock(&lock); + if (!rb) + return 101; + bpf_obj_drop(container_of(rb, struct tree_node, rb)); + } + if (less_calls < TRIES - 1) + return 102; + return less_spun; +} + +/* + * All registers are live at the may_goto, so it saves one of them in the stack. They are the same + * after 70000 iterations: the loop reads the clock. + */ +__naked __noinline __used +static long regs_across_may_goto(void) +{ + asm volatile ( + "r0 = 10;" + "r6 = 6;" + "r7 = 7;" + "r8 = 8;" + "r9 = 0;" + "1:" + "may_goto 2f;" + "r9 += 1;" + "if r9 < 70000 goto 1b;" + "2:" + "r0 += r1;" + "r0 += r2;" + "r0 += r3;" + "r0 += r4;" + "r0 += r5;" + "r0 += r6;" + "r0 += r7;" + "r0 += r8;" + "r0 += r9;" + "exit;" + ::: __clobber_all); +} + +SEC("tc") +__description("timed may_goto: registers across may_goto") +__success __retval(70046) +__naked void regs_across_may_goto_main(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call regs_across_may_goto;" + "exit;" + ::: __clobber_all); +} + +/* + * may_goto uses a register that is dead. In live_r0_rN() R0 - RN are live at the may_goto and the + * next one is not. The values come from memory, so the verifier doesn't know them. Return their sum + * and the number of iterations. + */ +__u64 vals[10] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; +__u64 zero64; + +#define LD0(b) "r0 = *(u64 *)(r" #b " + 0);" +#define LD1(b) LD0(b) "r1 = *(u64 *)(r" #b " + 8);" +#define LD2(b) LD1(b) "r2 = *(u64 *)(r" #b " + 16);" +#define LD3(b) LD2(b) "r3 = *(u64 *)(r" #b " + 24);" +#define LD4(b) LD3(b) "r4 = *(u64 *)(r" #b " + 32);" +#define LD5(b) LD4(b) "r5 = *(u64 *)(r" #b " + 40);" +#define LD6(b) LD5(b) "r6 = *(u64 *)(r" #b " + 48);" +#define LD7(b) LD6(b) "r7 = *(u64 *)(r" #b " + 56);" +#define LD8(b) LD7(b) "r8 = *(u64 *)(r" #b " + 64);" + +#define SUM0 +#define SUM1 SUM0 "r0 += r1;" +#define SUM2 SUM1 "r0 += r2;" +#define SUM3 SUM2 "r0 += r3;" +#define SUM4 SUM3 "r0 += r4;" +#define SUM5 SUM4 "r0 += r5;" +#define SUM6 SUM5 "r0 += r6;" +#define SUM7 SUM6 "r0 += r7;" +#define SUM8 SUM7 "r0 += r8;" + +/* The count of iterations is at r10 - 8. It begins with a number from memory too. */ +#define COUNT_INIT(t) \ + "r" #t " = %[zero64] ll;" \ + "r" #t " = *(u64 *)(r" #t " + 0);" \ + "*(u64 *)(r10 - 8) = r" #t ";" +#define COUNT_AND_LOOP(t) \ + "r" #t " = *(u64 *)(r10 - 8);" \ + "r" #t " += 1;" \ + "*(u64 *)(r10 - 8) = r" #t ";" \ + "if r" #t " < 70000 goto 1b;" +#define ADD_COUNT_AND_EXIT \ + "r1 = *(u64 *)(r10 - 8);" \ + "r0 += r1;" \ + "exit;" \ + : \ + : __imm_addr(vals), __imm_addr(zero64) \ + : __clobber_all + +#define LIVE_REGS(n, next) \ +__naked __noinline __used \ +static long live_r0_r##n(void) \ +{ \ + asm volatile ( \ + COUNT_INIT(next) \ + "r" #n " = %[vals] ll;" \ + LD##n(n) \ + "1:" \ + "may_goto 2f;" \ + COUNT_AND_LOOP(next) \ + "2:" \ + SUM##n \ + ADD_COUNT_AND_EXIT); \ +} + +LIVE_REGS(0, 1) +LIVE_REGS(1, 2) +LIVE_REGS(2, 3) +LIVE_REGS(3, 4) +LIVE_REGS(4, 5) +LIVE_REGS(5, 6) +LIVE_REGS(6, 7) +LIVE_REGS(7, 8) +LIVE_REGS(8, 9) + +/* R2 is read at the top of the loop only: it is live at the may_goto through the back edge. */ +__naked __noinline __used +static long live_back_edge(void) +{ + asm volatile ( + COUNT_INIT(3) + "r2 = %[vals] ll;" + LD2(2) + "1:" + "*(u64 *)(r10 - 16) = r2;" + "may_goto 2f;" + COUNT_AND_LOOP(3) + "2:" + "r0 += r1;" + "r2 = *(u64 *)(r10 - 16);" + "r0 += r2;" + ADD_COUNT_AND_EXIT); +} + +/* R2 is read only where the may_goto jumps to. */ +__naked __noinline __used +static long live_target(void) +{ + asm volatile ( + COUNT_INIT(3) + "r2 = %[vals] ll;" + LD2(2) + "1:" + "may_goto 2f;" + COUNT_AND_LOOP(3) + "goto 3f;" + "2:" + "r0 += r2;" + "3:" + "r0 += r1;" + ADD_COUNT_AND_EXIT); +} + +/* R2 is set in front of the may_goto in each iteration, and read only right behind it. */ +__naked __noinline __used +static long live_fall_through(void) +{ + asm volatile ( + COUNT_INIT(3) + "r1 = %[vals] ll;" + LD1(1) + "*(u64 *)(r10 - 16) = r0;" + "1:" + "r2 = %[vals] ll;" + "r2 = *(u64 *)(r2 + 16);" + "may_goto 2f;" + "*(u64 *)(r10 - 16) = r2;" + COUNT_AND_LOOP(3) + "2:" + "r0 += r1;" + "r2 = *(u64 *)(r10 - 16);" + "r0 += r2;" + ADD_COUNT_AND_EXIT); +} + +#define CALL_AND_ADD(f) "call " #f ";" "r6 += r0;" +#define CALL_ALL_LIVE \ + "r6 = 0;" \ + CALL_AND_ADD(live_r0_r0) \ + CALL_AND_ADD(live_r0_r1) \ + CALL_AND_ADD(live_r0_r2) \ + CALL_AND_ADD(live_r0_r3) \ + CALL_AND_ADD(live_r0_r4) \ + CALL_AND_ADD(live_r0_r5) \ + CALL_AND_ADD(live_r0_r6) \ + CALL_AND_ADD(live_r0_r7) \ + CALL_AND_ADD(live_r0_r8) \ + CALL_AND_ADD(live_back_edge) \ + CALL_AND_ADD(live_target) \ + CALL_AND_ADD(live_fall_through) \ + "r0 = r6;" \ + "exit;" + +/* + * The sums: 1 + 3 + 6 + 10 + 15 + 21 + 28 + 36 + 45 = 165. Then 6, 3 (the may_goto doesn't jump) + * and 6. 12 * 70000 iterations: each function reads the clock. + */ +SEC("tc") +__description("timed may_goto: may_goto takes a register that is dead") +__success __retval(840180) +__naked void dead_reg(void) +{ + asm volatile ( + CALL_ALL_LIVE + ::: __clobber_all); +} + +/* No iteration: 165, then 6, 6 (the may_goto jumps) and 4 (R2 isn't stored). */ +SEC("tc") +__description("timed may_goto: may_goto takes a register that is dead, the time is up") +__success __retval(181) +__naked void dead_reg_time_is_up(void) +{ + asm volatile ( + "call spins_on_may_goto;" + CALL_ALL_LIVE + ::: __clobber_all); +} + +/* + * The subprog uses up the time of the prog and returns. With a time slice for each call the caller + * calls it 65535 times before it reads the clock. + */ +__naked __noinline __used +static unsigned long spins_on_may_goto(void) +{ + asm volatile ( + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* The first may_goto of the function finds that the time is up */ +SEC("tc") +__description("timed may_goto: registers across may_goto when the time is up") +__success __retval(46) +__naked void regs_when_time_is_up(void) +{ + asm volatile ( + "call spins_on_may_goto;" + "r1 = 1;" + "r2 = 2;" + "r3 = 3;" + "r4 = 4;" + "r5 = 5;" + "call regs_across_may_goto;" + "exit;" + ::: __clobber_all); +} + +/* The first insn of the main prog is the target of a jump. It must not set the count again. */ +SEC("socket") +__description("timed may_goto: 9 insns") +__success __retval(1) +__naked void loop_calls_subprog_with_may_goto(void) +{ + asm volatile ( + "1:" + "call spins_on_may_goto;" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +/* + * These subprogs make 60000 iterations in each call. With a read of the clock after 65535 + * iterations of a call none of them reads it, and their caller does it after + * 65535 * 60000 * 60000 iterations. + */ +__naked __noinline __used +static unsigned long loops_60000(void) +{ + asm volatile ( + "r1 = 60000;" + "1:" + "may_goto 2f;" + "r1 += -1;" + "if r1 != 0 goto 1b;" + "2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +__naked __noinline __used +static unsigned long calls_60000_times(void) +{ + asm volatile ( + "r6 = 60000;" + "1:" + "call loops_60000;" + "may_goto 2f;" + "r6 += -1;" + "if r6 != 0 goto 1b;" + "2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("timed may_goto: three levels of loops that end before they read the clock") +__success __retval(1) +__naked void three_levels_of_short_loops(void) +{ + asm volatile ( + "1:" + "call calls_60000_times;" + "may_goto 2f;" + "goto 1b;" + "2:" + "r0 = 1;" + "exit;" + ::: __clobber_all); +} + +/* The callee ends one iteration before it reads the clock. */ +__naked __noinline __used +static unsigned long loops_65534(void) +{ + asm volatile ( + "r1 = 65534;" + "1:" + "may_goto 2f;" + "r1 += -1;" + "if r1 != 0 goto 1b;" + "2:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("timed may_goto: the subprog makes 65534 iterations") +__success __retval(1) +__naked void loop_calls_subprog_with_65534_iterations(void) +{ + asm volatile ( + "call %[bpf_ktime_get_ns];" + "r6 = r0;" + "1:" + "call loops_65534;" + "may_goto 2f;" + "goto 1b;" + "2:" + "call %[bpf_ktime_get_ns];" + "r0 -= r6;" + "r1 = r0;" + "r0 = 1;" + "if r1 < 1000000000 goto +1;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_ktime_get_ns) + : __clobber_all); +} + +/* The subprog returns 16 bytes in R0 and R2. may_goto must not change them. */ +__naked __noinline __used +static unsigned __int128 returns_pair_after_may_goto(void) +{ + asm volatile ( + "r0 = 1;" + "r2 = 2;" + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("timed may_goto: the subprog returns 16 bytes") +__success __retval(3) +__naked void subprog_with_may_goto_returns_pair(void) +{ + asm volatile ( + "call returns_pair_after_may_goto;" + "r0 += r2;" + "exit;" + ::: __clobber_all); +} + +/* Dead code doesn't count: a prog that has no may_goto left gets no insns. */ +__naked __noinline __used +static unsigned long may_goto_is_dead_code(void) +{ + asm volatile ( + "r0 = 0;" + "if r0 == 0 goto 2f;" + "1:" + "may_goto 2f;" + "goto 1b;" + "2:" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("timed may_goto: may_goto in dead code") +__success __retval(0) +__xlated("0: call pc+1") +__xlated("1: exit") +__xlated("2: r0 = 0") +__xlated("3: exit") +__naked void dead_may_goto(void) +{ + asm volatile ( + "call may_goto_is_dead_code;" + "exit;" + ::: __clobber_all); +} + +static __noinline int throws_after_spin(void) +{ + spin(); + bpf_throw(0); + return 0; +} + +__noinline int exception_cb_loops(u64 cookie) +{ + return counts_to_1000(); +} + +/* The exception callback belongs to the invocation that throws. */ +SEC("tc") +__description("timed may_goto: bpf_throw() the exception callback has a loop") +__exception_cb(exception_cb_loops) +__success __retval(0) +int throw_after_timeout(void *ctx) +{ + return throws_after_spin(); +} + +#else + +SEC("socket") +__description("timed may_goto: not supported dummy test") +__success +int dummy_test(void) +{ + return 0; +} + +#endif diff --git a/tools/testing/selftests/bpf/progs/timed_may_goto_ext.c b/tools/testing/selftests/bpf/progs/timed_may_goto_ext.c new file mode 100644 index 000000000000..969ca58b6eed --- /dev/null +++ b/tools/testing/selftests/bpf/progs/timed_may_goto_ext.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +char _license[] SEC("license") = "GPL"; + +static __noinline int spins(void) +{ + __u64 t0 = bpf_ktime_get_ns(); + + while (can_loop) + ; + return bpf_ktime_get_ns() - t0 > 100 * 1000 * 1000; +} + +/* Each call of an extension prog is an invocation: one of its two calls spins. */ +SEC("freplace") +int replacement(int i) +{ + return spins() + spins() + i; +} diff --git a/tools/testing/selftests/bpf/progs/timed_may_goto_user.c b/tools/testing/selftests/bpf/progs/timed_may_goto_user.c new file mode 100644 index 000000000000..8b85815d45d2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/timed_may_goto_user.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0 +/* The tests of timed_may_goto.c that need help from user space. */ +#include +#include +#include +#include "bpf_misc.h" +#include "bpf_experimental.h" + +char _license[] SEC("license") = "GPL"; + +#define TRIES 4 +#define CLOCK_MONOTONIC 1 + +int zero; + +/* Return 1 if the loop went on until the time was up. */ +static __always_inline int spin(void) +{ + __u64 t0 = bpf_ktime_get_ns(); + + while (can_loop) + ; + return bpf_ktime_get_ns() - t0 > 100 * 1000 * 1000; +} + +static __noinline int spins(void) +{ + return spin(); +} + +static __noinline int counts_to_1000(void) +{ + int i, cnt = zero; + + for (i = zero; i < 1000 && can_loop; i++) + cnt++; + return cnt; +} + +/* Nothing special: the loops go on, then the time is up one time. */ +SEC("tc") +int normal_prog(void *ctx) +{ + int i, n = 0; + + if (counts_to_1000() != 1000) + return 100; + for (i = 0; i < TRIES; i++) + n += spins(); + return n; +} + +/* + * Async callbacks + */ +struct elem { + struct bpf_timer timer; + struct bpf_wq wq; + struct bpf_task_work tw; + struct bpf_rcu_head rcu; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 2); + __type(key, int); + __type(value, struct elem); +} async SEC(".maps"); + +int timer_cnt, wq_cnt, tw_cnt, rcu_cnt, rcu_trace_cnt; + +static int timer_cb(void *map, int *key, struct elem *e) +{ + timer_cnt = counts_to_1000(); + return 0; +} + +static int wq_cb(void *map, int *key, void *val) +{ + wq_cnt = counts_to_1000(); + return 0; +} + +static int tw_cb(struct bpf_map *map, void *key, void *val) +{ + tw_cnt = counts_to_1000(); + return 0; +} + +static int rcu_cb(struct bpf_map *map, void *key, void *val) +{ + rcu_cnt = counts_to_1000(); + return 0; +} + +static int rcu_trace_cb(struct bpf_map *map, void *key, void *val) +{ + rcu_trace_cnt = counts_to_1000(); + return 0; +} + +/* + * The time of this prog is up when it starts the callbacks. User space keeps the task on the cpu, + * so the callbacks find what it has left. Each call of a callback is an invocation of its own. + */ +SEC("syscall") +int async_after_timeout(void *ctx) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct elem *e, *e1; + int key = 0, key1 = 1; + + e = bpf_map_lookup_elem(&async, &key); + e1 = bpf_map_lookup_elem(&async, &key1); + if (!e || !e1) + return 100; + if (bpf_timer_init(&e->timer, &async, CLOCK_MONOTONIC) || + bpf_timer_set_callback(&e->timer, timer_cb)) + return 101; + if (bpf_wq_init(&e->wq, &async, 0) || bpf_wq_set_callback(&e->wq, wq_cb, 0)) + return 102; + if (!spins()) + return 103; + /* control: the time is up */ + if (counts_to_1000()) + return 109; + if (bpf_timer_start(&e->timer, 0, BPF_F_TIMER_CPU_PIN)) + return 104; + if (bpf_wq_start(&e->wq, 0)) + return 105; + if (bpf_task_work_schedule_resume(task, &e->tw, &async, tw_cb)) + return 106; + if (bpf_call_rcu(&e->rcu, &async, rcu_cb)) + return 107; + if (bpf_call_rcu_tasks_trace(&e1->rcu, &async, rcu_trace_cb)) + return 108; + return 0; +} + +int ticks; + +static int tick_cb(void *map, int *key, struct elem *e) +{ + if (counts_to_1000() != 1000) + ticks = 1000; + if (++ticks < 100) + bpf_timer_start(&e->timer, 20 * 1000 * 1000, BPF_F_TIMER_CPU_PIN); + return 0; +} + +/* + * A callback of the prog interrupts it, on the same cpu, each 20 ms. It has its own time. That + * doesn't give more time to the prog. Return how long the loop went on, in ms. + */ +SEC("syscall") +int spin_under_timer(void *ctx) +{ + struct elem *e; + int key = 1; + __u64 t0; + + e = bpf_map_lookup_elem(&async, &key); + if (!e) + return 0; + if (bpf_timer_init(&e->timer, &async, CLOCK_MONOTONIC) || + bpf_timer_set_callback(&e->timer, tick_cb) || + bpf_timer_start(&e->timer, 20 * 1000 * 1000, BPF_F_TIMER_CPU_PIN)) + return 0; + t0 = bpf_ktime_get_ns(); + while (can_loop) + ; + return (bpf_ktime_get_ns() - t0) / 1000000; +} + +/* + * Extension prog + */ +__noinline int replaced_global(int i) +{ + return spin() + i; +} + +SEC("tc") +int freplace_target(void *ctx) +{ + int i, n = 0; + + for (i = 0; i < TRIES; i++) + n += replaced_global(zero); + return n; +} + +/* + * Tail calls + */ +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); +} jmp_table SEC(".maps"); + +int tail_callee_spun; + +SEC("tc") +int tail_callee(struct __sk_buff *skb) +{ + int i; + + tail_callee_spun = 0; + for (i = 0; i < TRIES; i++) + tail_callee_spun += spins(); + return 0; +} + +static __noinline int tail_calls(struct __sk_buff *skb) +{ + bpf_tail_call(skb, &jmp_table, 0); + return zero + 100; +} + +/* The target of the tail call is another prog. It returns to this function. */ +SEC("tc") +int tail_call_in_subprog(struct __sk_buff *skb) +{ + int i, n; + + n = tail_calls(skb); + if (counts_to_1000() != 1000) + return 101; + for (i = 0; i < TRIES; i++) + n += spins(); + return n; +} + +SEC("tc") +int tail_call_after_timeout(struct __sk_buff *skb) +{ + spins(); + bpf_tail_call(skb, &jmp_table, 0); + return 100; +} + +static __noinline int throws(void) +{ + spin(); + bpf_throw(7); + return 0; +} + +SEC("tc") +int throw_after_timeout(void *ctx) +{ + return throws(); +} + +/* + * bpf_user_ringbuf_drain() + */ +struct { + __uint(type, BPF_MAP_TYPE_USER_RINGBUF); + __uint(max_entries, 4096); +} user_ringbuf SEC(".maps"); + +static long drain_cb(struct bpf_dynptr *dynptr, int *spun) +{ + *spun += spin(); + return 0; +} + +SEC("syscall") +int drain(void *ctx) +{ + int spun = 0; + + return bpf_user_ringbuf_drain(&user_ringbuf, drain_cb, &spun, 0) * 10 + spun; +} + +/* + * A tracing prog with large frames: they are in a private stack. + */ +int priv_stack_spun, priv_stack_cb_spun; + +static __noinline int spins_with_stack(void) +{ + volatile char buf[128] = {}; + + return spin() + buf[zero & 127]; +} + +static int priv_stack_cb(__u64 i, int *spun) +{ + volatile char buf[128] = {}; + + *spun += spin() + buf[zero & 127]; + return 0; +} + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(priv_stack) +{ + volatile char buf[128] = {}; + int i, n = buf[zero & 127], spun = 0; + + for (i = 0; i < TRIES; i++) + n += spins_with_stack(); + bpf_loop(TRIES, priv_stack_cb, &spun, zero); + priv_stack_spun = n; + priv_stack_cb_spun = spun; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index e0c389102db9..d908298b9fc7 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -903,26 +903,28 @@ __naked void unconditional_jump_to_fill(void) SEC("raw_tp") __arch_x86_64 __log_level(4) -__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 24") -/* may_goto counter at -24 */ -__xlated("0: *(u64 *)(r10 -24) =") -/* may_goto timestamp at -16 */ -__xlated("1: *(u64 *)(r10 -16) =") -__xlated("2: r1 = 1") +__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 8") +/* may_goto counter and timestamp */ +__xlated("0: r0 = ") +__xlated("2: r0 = &(void __percpu *)(r0)") __xlated("...") -__xlated("4: r0 = &(void __percpu *)(r0)") +__xlated("6: r1 = 1") __xlated("...") -/* may_goto expansion starts */ -__xlated("6: r12 = *(u64 *)(r10 -24)") -__xlated("7: if r12 == 0x0 goto pc+6") -__xlated("8: r12 -= 1") -__xlated("9: if r12 != 0x0 goto pc+2") -__xlated("10: r12 = -24") -__xlated("11: call unknown") -__xlated("12: *(u64 *)(r10 -24) = r12") +__xlated("8: r0 = &(void __percpu *)(r0)") +__xlated("...") +/* may_goto expansion starts, r0 and r1 are live */ +__xlated("10: r2 = ") +__xlated("12: r2 = &(void __percpu *)(r2)") +__xlated("13: r12 = *(u64 *)(r2 +0)") +__xlated("14: if r12 == 0x0 goto pc+6") +__xlated("15: r12 -= 1") +__xlated("16: if r12 != 0x0 goto pc+2") +__xlated("17: r12 = r2") +__xlated("18: call unknown") +__xlated("19: *(u64 *)(r2 +0) = r12") /* may_goto expansion ends */ -__xlated("13: *(u64 *)(r10 -8) = r1") -__xlated("14: exit") +__xlated("20: *(u64 *)(r10 -8) = r1") +__xlated("21: exit") __success __naked void may_goto_interaction_x86_64(void) { @@ -946,24 +948,26 @@ __arch_arm64 __arch_riscv64 __arch_loongarch __log_level(4) -__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 24") -/* may_goto counter at -24 */ -__xlated("0: *(u64 *)(r10 -24) =") -/* may_goto timestamp at -16 */ -__xlated("1: *(u64 *)(r10 -16) =") -__xlated("2: r1 = 1") -__xlated("3: call bpf_get_smp_processor_id") -/* may_goto expansion starts */ -__xlated("4: r12 = *(u64 *)(r10 -24)") -__xlated("5: if r12 == 0x0 goto pc+6") -__xlated("6: r12 -= 1") -__xlated("7: if r12 != 0x0 goto pc+2") -__xlated("8: r12 = -24") -__xlated("9: call unknown") -__xlated("10: *(u64 *)(r10 -24) = r12") +__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 8") +/* may_goto counter and timestamp */ +__xlated("0: r0 = ") +__xlated("2: r0 = &(void __percpu *)(r0)") +__xlated("...") +__xlated("6: r1 = 1") +__xlated("7: call bpf_get_smp_processor_id") +/* may_goto expansion starts, r0 and r1 are live */ +__xlated("8: r2 = ") +__xlated("10: r2 = &(void __percpu *)(r2)") +__xlated("11: r12 = *(u64 *)(r2 +0)") +__xlated("12: if r12 == 0x0 goto pc+6") +__xlated("13: r12 -= 1") +__xlated("14: if r12 != 0x0 goto pc+2") +__xlated("15: r12 = r2") +__xlated("16: call unknown") +__xlated("17: *(u64 *)(r2 +0) = r12") /* may_goto expansion ends */ -__xlated("11: *(u64 *)(r10 -8) = r1") -__xlated("12: exit") +__xlated("18: *(u64 *)(r10 -8) = r1") +__xlated("19: exit") __success __naked void may_goto_interaction(void) { diff --git a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c index 1fbcc5228306..4dd98acc6436 100644 --- a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c +++ b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c @@ -553,7 +553,11 @@ int cond_break5(const void *ctx) { int cnt1 = zero, cnt2; - for (;;) { + /* + * There is one time slice for all may_goto of the prog. When it's used up a function that + * is called after that makes no iteration. So this loop has an end. + */ + for (; cnt1 < 100;) { cond_break; cnt1++; } diff --git a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c index faf9eec0a63e..20152a15ca36 100644 --- a/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c +++ b/tools/testing/selftests/bpf/progs/verifier_may_goto_1.c @@ -82,23 +82,31 @@ __naked void may_goto_batch_1(void) SEC("raw_tp") __description("may_goto batch with offsets 2/0") +__load_if_JITed() __arch_x86_64 __arch_s390x __arch_arm64 __arch_riscv64 __arch_loongarch -__xlated("0: *(u64 *)(r10 -16) = 65535") -__xlated("1: *(u64 *)(r10 -8) = 0") -__xlated("2: r12 = *(u64 *)(r10 -16)") -__xlated("3: if r12 == 0x0 goto pc+6") -__xlated("4: r12 -= 1") -__xlated("5: if r12 != 0x0 goto pc+2") -__xlated("6: r12 = -16") -__xlated("7: call unknown") -__xlated("8: *(u64 *)(r10 -16) = r12") -__xlated("9: r0 = 1") -__xlated("10: r0 = 2") -__xlated("11: exit") +/* the main prog sets count, timestamp, owner */ +__xlated("0: r0 = ") +__xlated("2: r0 = &(void __percpu *)(r0)") +__xlated("3: *(u64 *)(r0 +0) = 65535") +__xlated("4: *(u64 *)(r0 +8) = 0") +__xlated("5: *(u64 *)(r0 +16) = 0") +/* may_goto, r0 is dead */ +__xlated("6: r0 = ") +__xlated("8: r0 = &(void __percpu *)(r0)") +__xlated("9: r12 = *(u64 *)(r0 +0)") +__xlated("10: if r12 == 0x0 goto pc+6") +__xlated("11: r12 -= 1") +__xlated("12: if r12 != 0x0 goto pc+2") +__xlated("13: r12 = r0") +__xlated("14: call unknown") +__xlated("15: *(u64 *)(r0 +0) = r12") +__xlated("16: r0 = 1") +__xlated("17: r0 = 2") +__xlated("18: exit") __success __naked void may_goto_batch_2(void) {