diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 228a3f6889c2..b43eb97bcd44 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -969,6 +969,10 @@ struct bpf_subprog_info { bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; + /* bpf_loop() or bpf_for_each_map_elem() call it in a loop */ + bool is_loop_cb: 1; + /* BPF code, another helper or a kfunc call it */ + bool is_called: 1; bool args_cached: 1; /* true if the return value is passed in the R0:R2 register pair */ bool ret_reg_pair: 1; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 66b9d1d9e961..a656634fa9e4 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2771,9 +2771,12 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) int stack_slots = subprogs[i].stack_extra / 8; /* an invocation begins, see bpf_timed_may_goto_save() for async callbacks */ bool main_prog = tmg && !i; + /* only a helper calls it, in a loop that ends when it returns 1 */ + bool loop_cb = tmg && i && subprogs[i].is_loop_cb && !subprogs[i].is_called && + !subprogs[i].is_async_cb; int cnt = 0; - if (timed_may_goto ? !main_prog : !stack_slots) + if (timed_may_goto ? !main_prog && !loop_cb : !stack_slots) continue; if (stack_slots > 1) { verifier_bug(env, "stack_slots supports may_goto only"); @@ -2793,10 +2796,23 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[cnt++] = ld_tmg[0]; insn_buf[cnt++] = ld_tmg[1]; insn_buf[cnt++] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0); - insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, count_off, - BPF_MAX_TIMED_LOOPS); - insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, timestamp_off, 0); - insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, owner_off, 0); + if (main_prog) { + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, count_off, + BPF_MAX_TIMED_LOOPS); + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, timestamp_off, 0); + insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, owner_off, 0); + } else { + /* + * The time is up, but the helper would call the function again and + * again. End its loop. The verifier has seen that: the loop can end + * after each call. + */ + insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, + count_off); + insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2); + insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, 1); + insn_buf[cnt++] = BPF_EXIT_INSN(); + } } else { /* Add ST insn to subprog prologue to init extra stack */ insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth, diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index fb96c61ebec0..5e4700fda66b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11508,6 +11508,12 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins if (err) return err; + if (!bpf_pseudo_kfunc_call(insn) && + (insn->imm == BPF_FUNC_loop || insn->imm == BPF_FUNC_for_each_map_elem)) + env->subprog_info[subprog].is_loop_cb = true; + else + env->subprog_info[subprog].is_called = true; + /* for callback functions enqueue entry to callback and * proceed with next instruction within current frame. */ @@ -11544,6 +11550,7 @@ static int check_static_func_call(struct bpf_verifier_env *env, int subprog, int err; caller = state->frame[state->curframe]; + env->subprog_info[subprog].is_called = true; /* * Track caller's total stack arg count (incoming + max outgoing).