mirror of
https://github.com/torvalds/linux.git
synced 2026-11-06 21:58:01 +01:00
bpf: One may_goto time slice per prog invocation
The count and the timestamp of may_goto are in the stack frame. So the
limit of 250 ms is for one call of a subprog, not for one invocation of the
prog, and loops that nest across calls multiply it. These 9 insns cause a
soft lockup:
main: sub:
1: call sub 1: may_goto 2f
may_goto 2f goto 1b
goto 1b 2: r0 = 0
2: r0 = 1 exit
exit
Use one count and one timestamp for all may_goto of a prog, in per-cpu
memory of the prog. The main prog sets them in its first insns. When the
time is up each loop ends at its next may_goto, in any function.
rX = address of the count
rX = &(void __percpu *)(rX)
r12 = *(u64 *)(rX +0)
if r12 == 0x0 goto target
r12 -= 1
if r12 != 0x0 goto pc+2
r12 = rX
call arch_bpf_timed_may_goto
*(u64 *)(rX +0) = r12
rX is a register that is dead at the may_goto. When all ten are live R0 is
saved in the stack.
An async callback can interrupt the prog on the same cpu. So its caller
sets count and timestamp, and puts the old values back at the end. A second
invocation of the prog on the same cpu gives more time to the first one.
x86-64 (Zen 5), nsec before after
iteration, empty loop 0.36 0.44
iteration, the body takes 4 cycles 0.89 0.89
call of a subprog and 1 iteration 1.21 1.16
The 56 progs of scx with may_goto get 1.7% more JITed code and use 25% less
stack.
Fixes: e723608bf4 ("bpf: Add verifier support for timed may_goto")
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://patch.msgid.link/20261031215543.682284-8-alexei.starovoitov@gmail.com
This commit is contained in:
parent
78821f4d8d
commit
ec274bb466
|
|
@ -1926,6 +1926,8 @@ struct bpf_prog_aux {
|
|||
const char *attach_func_name;
|
||||
struct bpf_prog **func;
|
||||
struct bpf_prog_aux *main_prog_aux;
|
||||
/* main prog: there when the prog has timed may_goto */
|
||||
struct bpf_timed_may_goto __percpu *timed_may_goto;
|
||||
void *jit_data; /* JIT specific data. arch dependent */
|
||||
struct bpf_jit_poke_descriptor *poke_tab;
|
||||
struct bpf_kfunc_desc_tab *kfunc_tab;
|
||||
|
|
|
|||
|
|
@ -758,9 +758,15 @@ struct bpf_prog_stats {
|
|||
struct u64_stats_sync syncp;
|
||||
} __aligned(2 * sizeof(u64));
|
||||
|
||||
/*
|
||||
* Per-cpu memory of a prog. All may_goto of one invocation of the prog use it. An invocation is
|
||||
* one call from the kernel into the main prog, with the subprogs and sync callbacks it reaches.
|
||||
*/
|
||||
struct bpf_timed_may_goto {
|
||||
u64 count;
|
||||
u64 timestamp;
|
||||
/* the async callback that has set count and timestamp, see bpf_timed_may_goto_save() */
|
||||
void *owner;
|
||||
};
|
||||
|
||||
struct sk_filter {
|
||||
|
|
@ -1328,6 +1334,8 @@ u64 bpf_arch_uaddress_limit(void);
|
|||
void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
|
||||
u64 arch_bpf_timed_may_goto(void);
|
||||
u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *);
|
||||
void bpf_timed_may_goto_save(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved);
|
||||
void bpf_timed_may_goto_restore(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved);
|
||||
bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);
|
||||
|
||||
static inline bool bpf_dump_raw_ok(const struct cred *cred)
|
||||
|
|
|
|||
|
|
@ -294,6 +294,7 @@ void __bpf_prog_free(struct bpf_prog *fp)
|
|||
mutex_destroy(&fp->aux->used_maps_mutex);
|
||||
mutex_destroy(&fp->aux->dst_mutex);
|
||||
kfree(fp->aux->poke_tab);
|
||||
free_percpu(fp->aux->timed_may_goto);
|
||||
kfree(fp->aux);
|
||||
}
|
||||
free_percpu(fp->stats);
|
||||
|
|
@ -3645,6 +3646,38 @@ u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p)
|
|||
return BPF_MAX_TIMED_LOOPS;
|
||||
}
|
||||
|
||||
/*
|
||||
* The main prog sets count and timestamp in its first insns. A call of an async callback is an
|
||||
* invocation of its own too, but it can interrupt an invocation of the prog on this cpu, or go on
|
||||
* next to one that is preempted. So its caller sets count and timestamp, and puts the old ones
|
||||
* back at the end. Not when another invocation has set them in the meantime: they are for that
|
||||
* one now.
|
||||
* The caller has disabled migration.
|
||||
*/
|
||||
void bpf_timed_may_goto_save(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved)
|
||||
{
|
||||
struct bpf_timed_may_goto *cur;
|
||||
|
||||
if (!prog->aux->timed_may_goto)
|
||||
return;
|
||||
cur = this_cpu_ptr(prog->aux->timed_may_goto);
|
||||
*saved = *cur;
|
||||
cur->count = BPF_MAX_TIMED_LOOPS;
|
||||
cur->timestamp = 0;
|
||||
cur->owner = saved;
|
||||
}
|
||||
|
||||
void bpf_timed_may_goto_restore(const struct bpf_prog *prog, struct bpf_timed_may_goto *saved)
|
||||
{
|
||||
struct bpf_timed_may_goto *cur;
|
||||
|
||||
if (!prog->aux->timed_may_goto)
|
||||
return;
|
||||
cur = this_cpu_ptr(prog->aux->timed_may_goto);
|
||||
if (cur->owner == saved)
|
||||
*cur = *saved;
|
||||
}
|
||||
|
||||
/* for configs without MMU or 32-bit */
|
||||
__weak const struct bpf_map_ops arena_map_ops;
|
||||
__weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena)
|
||||
|
|
|
|||
|
|
@ -1803,6 +1803,50 @@ static int may_goto_expand(struct bpf_insn *insn_buf, int off, int stack_off,
|
|||
return cnt + tail_cnt;
|
||||
}
|
||||
|
||||
/*
|
||||
* Timed may_goto. All may_goto of a prog use one count and one timestamp: per-cpu memory of the
|
||||
* prog, set in the first insns of the main prog, and by the callers of async callbacks.
|
||||
* @reg holds the address. It is a register that is dead at the may_goto, or R0 that is saved at
|
||||
* @save_off in the stack.
|
||||
*/
|
||||
static int may_goto_expand_timed(struct bpf_insn *insn_buf, int off,
|
||||
struct bpf_timed_may_goto __percpu *tmg, int reg, int save_off)
|
||||
{
|
||||
struct bpf_insn ld_tmg[] = { BPF_LD_IMM64(reg, (__force unsigned long)tmg) };
|
||||
int cnt = 0, jmp, tail_cnt = save_off ? 6 : 5;
|
||||
|
||||
if (save_off)
|
||||
insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_10, reg, save_off);
|
||||
insn_buf[cnt++] = ld_tmg[0];
|
||||
insn_buf[cnt++] = ld_tmg[1];
|
||||
insn_buf[cnt++] = BPF_MOV64_PERCPU_REG(reg, reg);
|
||||
insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, reg, 0);
|
||||
if (save_off) {
|
||||
insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2);
|
||||
insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_10, save_off);
|
||||
jmp = cnt++;
|
||||
insn_buf[jmp] = BPF_JMP32_A(off >= 0 ? off + tail_cnt : off - jmp);
|
||||
} else {
|
||||
jmp = cnt;
|
||||
off = off >= 0 ? off + tail_cnt : off - jmp;
|
||||
if (off == (s16)off) {
|
||||
insn_buf[cnt++] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, off);
|
||||
} else {
|
||||
insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 1);
|
||||
insn_buf[cnt++] = BPF_JMP32_A(off >= 0 ? off : off - 1);
|
||||
}
|
||||
}
|
||||
insn_buf[cnt++] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
|
||||
insn_buf[cnt++] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2);
|
||||
/* AX is the argument and the return value */
|
||||
insn_buf[cnt++] = BPF_MOV64_REG(BPF_REG_AX, reg);
|
||||
insn_buf[cnt++] = BPF_EMIT_CALL(arch_bpf_timed_may_goto);
|
||||
insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, reg, BPF_REG_AX, 0);
|
||||
if (save_off)
|
||||
insn_buf[cnt++] = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_10, save_off);
|
||||
return cnt;
|
||||
}
|
||||
|
||||
/* Do various post-verification rewrites in a single program pass.
|
||||
* These rewrites simplify JIT and interpreter implementations.
|
||||
*/
|
||||
|
|
@ -1825,6 +1869,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
|
|||
u16 stack_depth_extra = 0;
|
||||
/* arch_bpf_timed_may_goto() takes and returns AX in a cpu register: for JITs only */
|
||||
bool timed_may_goto = prog->jit_requested && bpf_jit_supports_timed_may_goto();
|
||||
struct bpf_timed_may_goto __percpu *tmg = NULL;
|
||||
|
||||
if (env->seen_exception && !env->exception_callback_subprog) {
|
||||
struct bpf_insn *patch = insn_buf;
|
||||
|
|
@ -2081,39 +2126,27 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
|
|||
}
|
||||
|
||||
if (bpf_is_may_goto_insn(insn) && timed_may_goto) {
|
||||
int stack_off_cnt = -stack_depth - 16;
|
||||
/*
|
||||
* AX is used as an argument to pass in the pointer to
|
||||
* count and timestamp, and also as the return value of
|
||||
* the call to arch_bpf_timed_may_goto.
|
||||
*/
|
||||
struct bpf_insn tail[] = {
|
||||
BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1),
|
||||
BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 3),
|
||||
BPF_MOV64_REG(BPF_REG_AX, BPF_REG_10),
|
||||
BPF_ALU64_IMM(BPF_ADD, BPF_REG_AX, stack_off_cnt),
|
||||
BPF_EMIT_CALL(arch_bpf_timed_may_goto),
|
||||
BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt),
|
||||
};
|
||||
u16 live = env->insn_aux_data[i + delta].live_regs_before;
|
||||
int reg, save_off = 0;
|
||||
|
||||
/*
|
||||
* Two 8 byte slots, depth-16 stores the count, and
|
||||
* depth-8 stores the start timestamp of the loop.
|
||||
*
|
||||
* The starting value of count is BPF_MAX_TIMED_LOOPS
|
||||
* (0xffff). Every iteration loads it and subs it by 1,
|
||||
* until the value becomes 0 in AX (thus, 1 in stack),
|
||||
* after which we call arch_bpf_timed_may_goto, which
|
||||
* either sets AX to 0xffff to keep looping, or to 0
|
||||
* upon timeout. AX is then stored into the stack. In
|
||||
* the next iteration, we either see 0 and break out, or
|
||||
* continue iterating until the next time value is 0
|
||||
* after subtraction, rinse and repeat.
|
||||
*/
|
||||
stack_depth_extra = 16;
|
||||
/* a register that is dead here holds the address of the count */
|
||||
for (reg = BPF_REG_0; reg <= BPF_REG_9; reg++)
|
||||
if (!(live & BIT(reg)))
|
||||
break;
|
||||
if (reg > BPF_REG_9) {
|
||||
reg = BPF_REG_0;
|
||||
save_off = -stack_depth - 8;
|
||||
stack_depth_extra = 8;
|
||||
}
|
||||
if (!tmg) {
|
||||
tmg = alloc_percpu_gfp(struct bpf_timed_may_goto,
|
||||
GFP_KERNEL_ACCOUNT);
|
||||
if (!tmg)
|
||||
return -ENOMEM;
|
||||
prog->aux->timed_may_goto = tmg;
|
||||
}
|
||||
prog->jit_required = 1;
|
||||
cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt,
|
||||
tail, ARRAY_SIZE(tail));
|
||||
cnt = may_goto_expand_timed(insn_buf, insn->off, tmg, reg, save_off);
|
||||
|
||||
new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
|
||||
if (!new_prog)
|
||||
|
|
@ -2734,24 +2767,36 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
|
|||
|
||||
env->prog->aux->stack_depth = subprogs[0].stack_depth;
|
||||
for (i = 0; i < env->subprog_cnt; i++) {
|
||||
int delta = timed_may_goto ? 2 : 1;
|
||||
int subprog_start = subprogs[i].start;
|
||||
int stack_slots = subprogs[i].stack_extra / 8;
|
||||
int slots = delta, cnt = 0;
|
||||
/* an invocation begins, see bpf_timed_may_goto_save() for async callbacks */
|
||||
bool main_prog = tmg && !i;
|
||||
int cnt = 0;
|
||||
|
||||
if (!stack_slots)
|
||||
if (timed_may_goto ? !main_prog : !stack_slots)
|
||||
continue;
|
||||
/* We need two slots in case timed may_goto is supported. */
|
||||
if (stack_slots > slots) {
|
||||
if (stack_slots > 1) {
|
||||
verifier_bug(env, "stack_slots supports may_goto only");
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
stack_depth = subprogs[i].stack_depth;
|
||||
if (timed_may_goto) {
|
||||
insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth,
|
||||
struct bpf_insn ld_tmg[] = {
|
||||
BPF_LD_IMM64(BPF_REG_0, (__force unsigned long)tmg)
|
||||
};
|
||||
int count_off = offsetof(struct bpf_timed_may_goto, count);
|
||||
int timestamp_off = offsetof(struct bpf_timed_may_goto, timestamp);
|
||||
int owner_off = offsetof(struct bpf_timed_may_goto, owner);
|
||||
|
||||
/* R0 has nothing at the entry */
|
||||
insn_buf[cnt++] = ld_tmg[0];
|
||||
insn_buf[cnt++] = ld_tmg[1];
|
||||
insn_buf[cnt++] = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);
|
||||
insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, count_off,
|
||||
BPF_MAX_TIMED_LOOPS);
|
||||
insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0);
|
||||
insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, timestamp_off, 0);
|
||||
insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_0, owner_off, 0);
|
||||
} else {
|
||||
/* Add ST insn to subprog prologue to init extra stack */
|
||||
insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth,
|
||||
|
|
@ -2778,7 +2823,7 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env)
|
|||
* to insn after BPF_ST that inits may_goto count.
|
||||
* Adjustment will succeed because bpf_patch_insn_data() didn't fail.
|
||||
*/
|
||||
WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));
|
||||
WARN_ON(adjust_jmp_off(env->prog, subprog_start, cnt - 1));
|
||||
}
|
||||
|
||||
/* Since poke tab is now finalized, publish aux to tracker. */
|
||||
|
|
|
|||
|
|
@ -1222,6 +1222,8 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer)
|
|||
{
|
||||
struct bpf_hrtimer *t = container_of(hrtimer, struct bpf_hrtimer, timer);
|
||||
struct bpf_map *map = READ_ONCE(t->cb.map);
|
||||
struct bpf_prog *prog = READ_ONCE(t->cb.prog);
|
||||
struct bpf_timed_may_goto saved;
|
||||
void *value = t->cb.value;
|
||||
bpf_callback_t callback_fn;
|
||||
void *key;
|
||||
|
|
@ -1229,7 +1231,7 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer)
|
|||
|
||||
BTF_TYPE_EMIT(struct bpf_timer);
|
||||
callback_fn = rcu_dereference_check(t->cb.callback_fn, rcu_read_lock_bh_held());
|
||||
if (!callback_fn || !map)
|
||||
if (!callback_fn || !map || !prog)
|
||||
goto out;
|
||||
|
||||
/* bpf_timer_cb() runs in hrtimer_run_softirq. It doesn't migrate and
|
||||
|
|
@ -1242,8 +1244,10 @@ static enum hrtimer_restart bpf_timer_cb(struct hrtimer *hrtimer)
|
|||
|
||||
key = map_key_from_value(map, value, &idx);
|
||||
|
||||
bpf_timed_may_goto_save(prog, &saved);
|
||||
callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0);
|
||||
/* The verifier checked that return value is zero. */
|
||||
bpf_timed_may_goto_restore(prog, &saved);
|
||||
|
||||
this_cpu_write(hrtimer_running, NULL);
|
||||
out:
|
||||
|
|
@ -1254,6 +1258,8 @@ static void bpf_wq_work(struct work_struct *work)
|
|||
{
|
||||
struct bpf_work *w = container_of(work, struct bpf_work, work);
|
||||
struct bpf_async_cb *cb = &w->cb;
|
||||
struct bpf_timed_may_goto saved;
|
||||
struct bpf_prog *prog;
|
||||
struct bpf_map *map;
|
||||
bpf_callback_t callback_fn;
|
||||
void *value = cb->value;
|
||||
|
|
@ -1267,14 +1273,17 @@ static void bpf_wq_work(struct work_struct *work)
|
|||
|
||||
map = READ_ONCE(cb->map);
|
||||
callback_fn = READ_ONCE(cb->callback_fn);
|
||||
if (!callback_fn || !map)
|
||||
prog = READ_ONCE(cb->prog);
|
||||
if (!callback_fn || !map || !prog)
|
||||
goto out;
|
||||
|
||||
key = map_key_from_value(map, value, &idx);
|
||||
|
||||
migrate_disable();
|
||||
|
||||
bpf_timed_may_goto_save(prog, &saved);
|
||||
callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0);
|
||||
bpf_timed_may_goto_restore(prog, &saved);
|
||||
|
||||
migrate_enable();
|
||||
out:
|
||||
|
|
@ -4611,6 +4620,7 @@ static void bpf_task_work_cancel(struct bpf_task_work_ctx *ctx,
|
|||
static void bpf_task_work_callback(struct callback_head *cb)
|
||||
{
|
||||
struct bpf_task_work_ctx *ctx = container_of(cb, struct bpf_task_work_ctx, work);
|
||||
struct bpf_timed_may_goto saved;
|
||||
enum bpf_task_work_state state;
|
||||
u32 idx;
|
||||
void *key;
|
||||
|
|
@ -4632,8 +4642,10 @@ static void bpf_task_work_callback(struct callback_head *cb)
|
|||
key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx);
|
||||
|
||||
migrate_disable();
|
||||
bpf_timed_may_goto_save(ctx->prog, &saved);
|
||||
ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key,
|
||||
(u64)(long)ctx->map_val, 0, 0);
|
||||
bpf_timed_may_goto_restore(ctx->prog, &saved);
|
||||
migrate_enable();
|
||||
|
||||
bpf_task_work_ctx_reset(ctx);
|
||||
|
|
@ -4873,6 +4885,7 @@ static void bpf_rcu_run_callback(struct rcu_head *rcu)
|
|||
struct bpf_rcu_head_kern *rh = container_of(rcu, struct bpf_rcu_head_kern, rcu);
|
||||
bpf_callback_t callback_fn = rh->callback_fn;
|
||||
struct bpf_prog *prog = rh->prog;
|
||||
struct bpf_timed_may_goto saved;
|
||||
struct bpf_map *map = rh->map;
|
||||
void *value, *key;
|
||||
u32 idx;
|
||||
|
|
@ -4884,7 +4897,9 @@ static void bpf_rcu_run_callback(struct rcu_head *rcu)
|
|||
smp_store_release(&rh->armed, 0);
|
||||
|
||||
rcu_read_lock_dont_migrate();
|
||||
bpf_timed_may_goto_save(prog, &saved);
|
||||
callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, 0, 0);
|
||||
bpf_timed_may_goto_restore(prog, &saved);
|
||||
rcu_read_unlock_migrate();
|
||||
|
||||
bpf_prog_put(prog);
|
||||
|
|
|
|||
|
|
@ -903,27 +903,28 @@ __naked void unconditional_jump_to_fill(void)
|
|||
SEC("raw_tp")
|
||||
__arch_x86_64
|
||||
__log_level(4)
|
||||
__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 24")
|
||||
/* may_goto counter at -24 */
|
||||
__xlated("0: *(u64 *)(r10 -24) =")
|
||||
/* may_goto timestamp at -16 */
|
||||
__xlated("1: *(u64 *)(r10 -16) =")
|
||||
__xlated("2: r1 = 1")
|
||||
__msg("subprog 0 (may_goto_interaction_x86_64) main {{.*}} stack 8")
|
||||
/* may_goto counter and timestamp */
|
||||
__xlated("0: r0 = ")
|
||||
__xlated("2: r0 = &(void __percpu *)(r0)")
|
||||
__xlated("...")
|
||||
__xlated("4: r0 = &(void __percpu *)(r0)")
|
||||
__xlated("6: r1 = 1")
|
||||
__xlated("...")
|
||||
/* may_goto expansion starts */
|
||||
__xlated("6: r12 = *(u64 *)(r10 -24)")
|
||||
__xlated("7: if r12 == 0x0 goto pc+7")
|
||||
__xlated("8: r12 -= 1")
|
||||
__xlated("9: if r12 != 0x0 goto pc+3")
|
||||
__xlated("10: r12 = r10")
|
||||
__xlated("11: r12 += -24")
|
||||
__xlated("12: call unknown")
|
||||
__xlated("13: *(u64 *)(r10 -24) = r12")
|
||||
__xlated("8: r0 = &(void __percpu *)(r0)")
|
||||
__xlated("...")
|
||||
/* may_goto expansion starts, r0 and r1 are live */
|
||||
__xlated("10: r2 = ")
|
||||
__xlated("12: r2 = &(void __percpu *)(r2)")
|
||||
__xlated("13: r12 = *(u64 *)(r2 +0)")
|
||||
__xlated("14: if r12 == 0x0 goto pc+6")
|
||||
__xlated("15: r12 -= 1")
|
||||
__xlated("16: if r12 != 0x0 goto pc+2")
|
||||
__xlated("17: r12 = r2")
|
||||
__xlated("18: call unknown")
|
||||
__xlated("19: *(u64 *)(r2 +0) = r12")
|
||||
/* may_goto expansion ends */
|
||||
__xlated("14: *(u64 *)(r10 -8) = r1")
|
||||
__xlated("15: exit")
|
||||
__xlated("20: *(u64 *)(r10 -8) = r1")
|
||||
__xlated("21: exit")
|
||||
__success
|
||||
__naked void may_goto_interaction_x86_64(void)
|
||||
{
|
||||
|
|
@ -947,25 +948,26 @@ __arch_arm64
|
|||
__arch_riscv64
|
||||
__arch_loongarch
|
||||
__log_level(4)
|
||||
__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 24")
|
||||
/* may_goto counter at -24 */
|
||||
__xlated("0: *(u64 *)(r10 -24) =")
|
||||
/* may_goto timestamp at -16 */
|
||||
__xlated("1: *(u64 *)(r10 -16) =")
|
||||
__xlated("2: r1 = 1")
|
||||
__xlated("3: call bpf_get_smp_processor_id")
|
||||
/* may_goto expansion starts */
|
||||
__xlated("4: r12 = *(u64 *)(r10 -24)")
|
||||
__xlated("5: if r12 == 0x0 goto pc+7")
|
||||
__xlated("6: r12 -= 1")
|
||||
__xlated("7: if r12 != 0x0 goto pc+3")
|
||||
__xlated("8: r12 = r10")
|
||||
__xlated("9: r12 += -24")
|
||||
__xlated("10: call unknown")
|
||||
__xlated("11: *(u64 *)(r10 -24) = r12")
|
||||
__msg("subprog 0 (may_goto_interaction) main {{.*}} stack 8")
|
||||
/* may_goto counter and timestamp */
|
||||
__xlated("0: r0 = ")
|
||||
__xlated("2: r0 = &(void __percpu *)(r0)")
|
||||
__xlated("...")
|
||||
__xlated("6: r1 = 1")
|
||||
__xlated("7: call bpf_get_smp_processor_id")
|
||||
/* may_goto expansion starts, r0 and r1 are live */
|
||||
__xlated("8: r2 = ")
|
||||
__xlated("10: r2 = &(void __percpu *)(r2)")
|
||||
__xlated("11: r12 = *(u64 *)(r2 +0)")
|
||||
__xlated("12: if r12 == 0x0 goto pc+6")
|
||||
__xlated("13: r12 -= 1")
|
||||
__xlated("14: if r12 != 0x0 goto pc+2")
|
||||
__xlated("15: r12 = r2")
|
||||
__xlated("16: call unknown")
|
||||
__xlated("17: *(u64 *)(r2 +0) = r12")
|
||||
/* may_goto expansion ends */
|
||||
__xlated("12: *(u64 *)(r10 -8) = r1")
|
||||
__xlated("13: exit")
|
||||
__xlated("18: *(u64 *)(r10 -8) = r1")
|
||||
__xlated("19: exit")
|
||||
__success
|
||||
__naked void may_goto_interaction(void)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -88,19 +88,25 @@ __arch_s390x
|
|||
__arch_arm64
|
||||
__arch_riscv64
|
||||
__arch_loongarch
|
||||
__xlated("0: *(u64 *)(r10 -16) = 65535")
|
||||
__xlated("1: *(u64 *)(r10 -8) = 0")
|
||||
__xlated("2: r12 = *(u64 *)(r10 -16)")
|
||||
__xlated("3: if r12 == 0x0 goto pc+7")
|
||||
__xlated("4: r12 -= 1")
|
||||
__xlated("5: if r12 != 0x0 goto pc+3")
|
||||
__xlated("6: r12 = r10")
|
||||
__xlated("7: r12 += -16")
|
||||
__xlated("8: call unknown")
|
||||
__xlated("9: *(u64 *)(r10 -16) = r12")
|
||||
__xlated("10: r0 = 1")
|
||||
__xlated("11: r0 = 2")
|
||||
__xlated("12: exit")
|
||||
/* the main prog sets count, timestamp, owner */
|
||||
__xlated("0: r0 = ")
|
||||
__xlated("2: r0 = &(void __percpu *)(r0)")
|
||||
__xlated("3: *(u64 *)(r0 +0) = 65535")
|
||||
__xlated("4: *(u64 *)(r0 +8) = 0")
|
||||
__xlated("5: *(u64 *)(r0 +16) = 0")
|
||||
/* may_goto, r0 is dead */
|
||||
__xlated("6: r0 = ")
|
||||
__xlated("8: r0 = &(void __percpu *)(r0)")
|
||||
__xlated("9: r12 = *(u64 *)(r0 +0)")
|
||||
__xlated("10: if r12 == 0x0 goto pc+6")
|
||||
__xlated("11: r12 -= 1")
|
||||
__xlated("12: if r12 != 0x0 goto pc+2")
|
||||
__xlated("13: r12 = r0")
|
||||
__xlated("14: call unknown")
|
||||
__xlated("15: *(u64 *)(r0 +0) = r12")
|
||||
__xlated("16: r0 = 1")
|
||||
__xlated("17: r0 = 2")
|
||||
__xlated("18: exit")
|
||||
__success
|
||||
__naked void may_goto_batch_2(void)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user