From 14eba6d98cabe625218fae0bfed8959d97226bc3 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 25 Aug 2026 10:35:57 +0800 Subject: [PATCH 01/69] sched_ext: Rename tg->scx.idle to tg->scx.sched_idle A bare "idle" reads as CPU idle state in sched_ext (ops.update_idle(), idle cpumasks, scx_bpf_pick_idle_cpu()). cpu.idle is the cgroup analog of the SCHED_IDLE policy, so name the task_group field sched_idle to match scx_cgroup_init_args.sched_idle. Pure rename, no behavior change. Signed-off-by: Tao Cui Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 2 +- kernel/sched/ext/ext.c | 8 ++++---- kernel/sched/ext/sub.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 582d7cd4a983..7b9cdb65deef 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -323,7 +323,7 @@ struct scx_task_group { u64 bw_period_us; u64 bw_quota_us; u64 bw_burst_us; - bool idle; + bool sched_idle; #endif }; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 901fb0f8b976..8041c87a3562 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4684,7 +4684,7 @@ void scx_tg_init(struct task_group *tg) tg->scx.weight = CGROUP_WEIGHT_DFL; tg->scx.bw_period_us = default_bw_period_us(); tg->scx.bw_quota_us = RUNTIME_INF; - tg->scx.idle = false; + tg->scx.sched_idle = false; } /** @@ -4767,7 +4767,7 @@ int scx_tg_online(struct task_group *tg) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle }; + .sched_idle = tg->scx.sched_idle }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, tg->css.cgroup, &args); @@ -4937,7 +4937,7 @@ void scx_group_set_idle(struct task_group *tg, bool idle) SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle); /* Update the task group's idle state */ - tg->scx.idle = idle; + tg->scx.sched_idle = idle; percpu_up_read(&scx_cgroup_ops_rwsem); } @@ -5188,7 +5188,7 @@ static int scx_cgroup_init(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; ret = SCX_CALL_OP_RET(sch, cgroup_init, NULL, css->cgroup, &args); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 385302d19914..a17d84db93bd 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1361,7 +1361,7 @@ static s32 scx_cgroup_claim_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; if (tg->scx.sched != parent || @@ -1465,7 +1465,7 @@ static void scx_cgroup_return_subtree(struct scx_sched *sch) .bw_period_us = tg->scx.bw_period_us, .bw_quota_us = tg->scx.bw_quota_us, .bw_burst_us = tg->scx.bw_burst_us, - .sched_idle = tg->scx.idle, + .sched_idle = tg->scx.sched_idle, }; /* the first pass must have transferred everything */ From 92344fad5b54d1c20ea19f882a4f086ca68c11c0 Mon Sep 17 00:00:00 2001 From: fangqiurong Date: Tue, 1 Sep 2026 09:32:58 +0800 Subject: [PATCH 02/69] sched_ext: Skip per-CPU data allocation for built-in DSQs Only user DSQs use the per-DSQ deferred reenqueue state: schedule_dsq_reenq() serves the local DSQ through sch->pcpu->deferred_reenq_local and rejects all other built-in DSQ ids. Every DSQ, built-in or not, still allocates a struct scx_dsq_pcpu per possible CPU. As the local, reject, rescue and bypass DSQs are themselves per-CPU, the wasted memory grows with the square of the CPU count, most of it at boot before any scheduler is loaded. Skip the allocation for built-in DSQ ids. While at it, rename ->pcpu to ->pcpu_user so that the per-CPU data reads as user-DSQ only. Signed-off-by: fangqiurong Reviewed-by: Zhan Xusheng Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 2 +- kernel/sched/ext/ext.c | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 7b9cdb65deef..8de69843c215 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -91,7 +91,7 @@ struct scx_dispatch_q { struct rhash_head hash_node; struct llist_node free_node; struct scx_sched *sched; - struct scx_dsq_pcpu __percpu *pcpu; + struct scx_dsq_pcpu __percpu *pcpu_user; struct rcu_head rcu; }; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 8041c87a3562..0e8ddc307ecb 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1127,7 +1127,7 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, } else if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) { rq = this_rq(); - struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu, cpu_of(rq)); + struct scx_dsq_pcpu *dsq_pcpu = per_cpu_ptr(dsq->pcpu_user, cpu_of(rq)); struct scx_deferred_reenq_user *dru = &dsq_pcpu->deferred_reenq_user; /* @@ -5051,12 +5051,16 @@ s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch) dsq->id = dsq_id; dsq->sched = sch; - dsq->pcpu = alloc_percpu(struct scx_dsq_pcpu); - if (!dsq->pcpu) + /* per-DSQ deferred reenq state is only needed for user DSQs */ + if (dsq_id & SCX_DSQ_FLAG_BUILTIN) + return 0; + + dsq->pcpu_user = alloc_percpu(struct scx_dsq_pcpu); + if (!dsq->pcpu_user) return -ENOMEM; for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); pcpu->dsq = dsq; INIT_LIST_HEAD(&pcpu->deferred_reenq_user.node); @@ -5069,8 +5073,11 @@ static void exit_dsq(struct scx_dispatch_q *dsq) { s32 cpu; + if (!dsq->pcpu_user) + return; + for_each_possible_cpu(cpu) { - struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu, cpu); + struct scx_dsq_pcpu *pcpu = per_cpu_ptr(dsq->pcpu_user, cpu); struct scx_deferred_reenq_user *dru = &pcpu->deferred_reenq_user; struct rq *rq = cpu_rq(cpu); @@ -5084,7 +5091,7 @@ static void exit_dsq(struct scx_dispatch_q *dsq) } } - free_percpu(dsq->pcpu); + free_percpu(dsq->pcpu_user); } static void free_dsq_rcufn(struct rcu_head *rcu) From 00d977d84226356fe766977d48bfbdf72169a281 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 2 Sep 2026 10:48:11 +0800 Subject: [PATCH 03/69] sched_ext: Document the rolling-cursor requirement for dsq_vtime scx_bpf_dsq_insert_vtime() orders tasks by time_before64(), which is only meaningful when the values within a given DSQ stay less than 2^63 apart. This is implicit in how a vtime scheduler works but not spelled out anywhere. Document it so BPF scheduler authors know the constraint. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0e8ddc307ecb..6a5c83aea047 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8910,7 +8910,9 @@ struct scx_bpf_dsq_insert_vtime_args { * * @args->vtime ordering is according to time_before64() which considers * wrapping. A numerically larger vtime may indicate an earlier position in the - * ordering and vice-versa. + * ordering and vice-versa. vtime is a rolling cursor and values used for + * ordering within a given DSQ should stay less than 2^63 apart for + * time_before64() ordering to remain well-defined. * * A DSQ can only be used as a FIFO or priority queue at any given time and this * function must not be called on a DSQ which already has one or more FIFO tasks From bf2f23dc5a36237c0d4738966829e5cbde973c8a Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 2 Sep 2026 10:48:12 +0800 Subject: [PATCH 04/69] sched_ext/scx_flatcg: Make cgv_node_less() wraparound-safe cgv_node_less() compares cvtimes with a plain <, which misorders once cvtime wraps: the wrapped node lands at the front of the tree while the unwrapped ones get stuck behind it. Each CPU picking a cgroup charges it a full slice, making wrap occur earlier than a naive estimate might suggest. Use time_before() instead. cgrp_cap_budget() bounds the lag behind cvtime_now, while the lead is bounded by the slice charge plus pending cvtime_delta on re-insertion, so the cyclic ordering assumptions of time_before() hold. Fixes: a4103eacc2ab ("sched_ext: Add a cgroup scheduler which uses flattened hierarchy") Reported-by: Sashiko Link: https://lore.kernel.org/r/3f1ce004-e259-4e72-a5f7-14a5050053bd@linux.dev Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- tools/sched_ext/scx_flatcg.bpf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index 454ebb820c5e..5e6abd0bccb3 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -144,7 +144,7 @@ static bool cgv_node_less(struct bpf_rb_node *a, const struct bpf_rb_node *b) cgc_a = container_of(a, struct cgv_node, rb_node); cgc_b = container_of(b, struct cgv_node, rb_node); - return cgc_a->cvtime < cgc_b->cvtime; + return time_before(cgc_a->cvtime, cgc_b->cvtime); } static struct fcg_cpu_ctx *find_cpu_ctx(void) From c659e506f9a77d7a0e100d3555bccd5a44babf0e Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Wed, 2 Sep 2026 17:36:11 +0800 Subject: [PATCH 05/69] sched_ext: Reject NMI calls to lock-taking kfuncs commit e06ece82d7b0 ("sched_ext: Report NMI kicks with scx_error()") made scx_bpf_kick_cpu() reject NMI calls, and its cover letter describes the reachability: sched_ext kfuncs in the "any" category "are callable from tracing progs that can attach to functions running in NMI", and an unlucky call from there "could deadlock the machine". The fix in that series made the error/exit path lock-free so scx_error() is safe to call from NMI. That closes the *error* path of every kfunc, but not a kfunc's own business-logic lock acquisition on its success path. The remaining lock-taking kfuncs that scx_kfunc_context_filter() exposes to BPF_PROG_TYPE_TRACING have the same hazard: if an NMI lands on a CPU whose interrupted context already holds the lock, the kfunc's raw spinlock acquisition spins forever and hard-locks the CPU: - scx_bpf_destroy_dsq() -> dsq->lock - scx_bpf_dsq_reenq() -> rq's deferred_reenq_lock - scx_bpf_cpuperf_set() / scx_bpf_cidperf_set() -> rq->lock - scx_bpf_sub_grant() / scx_bpf_sub_revoke() -> pshard lock (via the shared sub_cap_preamble()) - bpf_iter_scx_dsq_next() / bpf_iter_scx_dsq_destroy() -> dsq->lock (the reject is placed in bpf_iter_scx_dsq_new(), which is otherwise lockless, so next() and destroy() are no-ops while it leaves kit->dsq NULL) As things stand, there is no scenario for reenqueueing, iterating a DSQ, setting a performance target or granting sub-caps from NMI. The guards defend against a buggy or malicious BPF program turning an "any"-category kfunc into a machine-wide hard-lockup through the door that scx_kfunc_context_filter() already opens. This matches the intent of scx_bpf_kick_cpu()'s NMI check, which the commit cited above added not to enable an NMI use case but to surface such a bug as a clean abort. Route all of them through a new scx_kf_allowed_ctx() helper and reuse scx_bpf_kick_cpu()'s existing in_nmi() check - now shared with its cid equivalent scx_bpf_kick_cid() through scx_kick_cpu() - so the rule is stated once and the coverage is auditable from one place. scx_error() is already NMI-safe (commit f883dbb64ca5 ("sched_ext: Make exit claiming lock-free")), so the reject-abort cannot deadlock the lock acquisition. Kfuncs with an error return report -EDEADLK, the situation being avoided. Read-only members of the reachable sets (dsq_peek, dsq_nr_queued, cpuperf_cur/cap, sub_caps, the idle cpumask helpers and the cid lookups) take no scheduler lock on the path a tracing program reaches them, and were audited to that effect; they are correctly left unguarded. The select_cpu kfuncs do take pi_lock, but scx_kfunc_context_filter() only exposes the any/idle/cid sets to BPF_PROG_TYPE_TRACING, and struct_ops programs never run in NMI, so no lock-taking path here is reachable from NMI. Link: https://lore.kernel.org/r/20260901095652.1009104-1-liwanwu@kylinos.cn Signed-off-by: Wanwu Li Acked-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 30 +++++++++++++++++++++--------- kernel/sched/ext/internal.h | 16 ++++++++++++++++ kernel/sched/ext/sub.c | 3 +++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6a5c83aea047..387953b92668 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -9537,14 +9537,8 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) struct rq *this_rq; unsigned long irq_flags; - /* - * The per-cpu kick list is guarded only by local_irq_save(), which does - * not mask NMIs, so kicking from NMI could corrupt it and is unsupported. - */ - if (unlikely(in_nmi())) { - scx_error(sch, "scx_bpf_kick_cpu() called from NMI"); + if (!scx_kf_allowed_ctx(sch)) return; - } local_irq_save(irq_flags); @@ -9712,8 +9706,13 @@ __bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux) guard(rcu)(); sch = scx_prog_sched(aux); - if (sch) - destroy_dsq(sch, dsq_id); + if (unlikely(!sch)) + return; + + if (!scx_kf_allowed_ctx(sch)) + return; + + destroy_dsq(sch, dsq_id); } /** @@ -9750,6 +9749,9 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, if (unlikely(!sch)) return -ENODEV; + if (!scx_kf_allowed_ctx(sch)) + return -EDEADLK; + if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) return -EINVAL; @@ -9876,6 +9878,9 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, return; } + if (!scx_kf_allowed_ctx(sch)) + return; + /* not specifying any filter bits is the same as %SCX_REENQ_ANY */ if (!(reenq_flags & __SCX_REENQ_FILTER_MASK)) reenq_flags |= SCX_REENQ_ANY; @@ -10263,6 +10268,9 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au if (unlikely(!sch)) return; + if (!scx_kf_allowed_ctx(sch)) + return; + scx_cpuperf_set(sch, cpu, perf); } @@ -10288,6 +10296,10 @@ __bpf_kfunc s32 scx_bpf_cidperf_set(s32 cid, u32 perf, sch = scx_prog_sched(aux); if (unlikely(!sch)) return -ENODEV; + + if (!scx_kf_allowed_ctx(sch)) + return -EDEADLK; + cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return cpu; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 076a351bb3f2..4f876e8f0655 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2115,6 +2115,22 @@ extern struct scx_sched *scx_enabling_sub_sched; #define scx_error(sch, fmt, args...) \ scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) +/* + * Tracing progs can call kfuncs from NMI. Kfuncs that take scheduler locks or + * touch the kick lists, which are only protected by irq masking, can't run + * there, so abort the scheduler instead. scx_error() is NMI-safe. + */ +static __always_inline bool __scx_kf_allowed_ctx(struct scx_sched *sch, const char *who) +{ + if (unlikely(in_nmi())) { + scx_error(sch, "%s called from NMI", who); + return false; + } + return true; +} + +#define scx_kf_allowed_ctx(sch) __scx_kf_allowed_ctx((sch), __func__) + /** * scx_root_protected_live - Root sched for paths that only run while live * diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a17d84db93bd..380a5653dc52 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -2267,6 +2267,9 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * if (unlikely(!parent)) return -ENODEV; + if (!scx_kf_allowed_ctx(parent)) + return -EDEADLK; + if (!scx_is_cid_type()) { scx_error(parent, "sub-cap kfuncs require a cid-form scheduler"); return -EOPNOTSUPP; From 8848333264b73cb6617fc04e11c245ab1bd22b28 Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 3 Sep 2026 11:57:19 +0800 Subject: [PATCH 06/69] sched_ext: Make scx_locked_rq() return NULL from NMI scx_locked_rq() reads the per-CPU scx_locked_rq_state, which tracks the rq locked by the context running on the CPU. Tracing progs can call kfuncs from NMI, and an NMI interrupts - rather than replaces - the context that set scx_locked_rq_state, so a non-NULL read from NMI falsely tells the caller that it holds the interrupted context's rq lock. Three "any"-category kfuncs read scx_locked_rq() on their success path and take an unsafe fast path on a non-NULL return: - scx_bpf_task_set_slice() writes p->scx.slice directly, racing update_curr_scx()'s non-atomic read-modify-write of the same field. - scx_bpf_dsq_nr_queued() resolves %SCX_DSQ_LOCAL to (scx_locked_rq() ?: this_rq()) and can report the interrupted context's local DSQ length instead of the caller's. - scx_bpf_locked_rq() hands the interrupted context's rq to the BPF program, which may then operate on it as if it owned the rq lock. Make scx_locked_rq() return NULL from NMI so that all three take their unlocked paths: scx_bpf_task_set_slice() stashes the request into the atomic p->scx.slice_oob for application under the rq lock, scx_bpf_dsq_nr_queued() falls back to this_rq(), and scx_bpf_locked_rq() reports an error and aborts the scheduler through the NMI-safe exit path. The kfuncs that take scheduler locks reject NMI calls through scx_kf_allowed_ctx() before reaching scx_locked_rq(), and the internal callers only run from struct_ops callbacks, which never run in NMI, so no other caller is affected. Suggested-by: Tejun Heo Link: https://lore.kernel.org/r/d84b31727f04e1ed0d40042ba1c09e61@kernel.org Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/internal.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 4f876e8f0655..e1eb3a0d456c 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2221,6 +2221,15 @@ static inline void scx_schedule_reenq_local(struct rq *rq, u64 reenq_flags) */ static inline struct rq *scx_locked_rq(void) { + /* + * Tracing progs can call kfuncs from NMI. scx_locked_rq_state tracks + * the rq locked by the interrupted context, so a non-NULL read from + * NMI would falsely claim its lock. Return NULL from NMI so that + * callers take their unlocked paths. + */ + if (unlikely(in_nmi())) + return NULL; + return __this_cpu_read(scx_locked_rq_state); } From 8b9b3698796a6b99f43fd09de23bfa5431e0f07e Mon Sep 17 00:00:00 2001 From: Wanwu Li Date: Thu, 3 Sep 2026 11:57:54 +0800 Subject: [PATCH 07/69] sched_ext: Protect the idle-search scratch nodemask with irqsave pick_idle_cpu_from_online_nodes() uses the per-CPU per_cpu_unvisited nodemask as scratch while walking the online nodes, protected only by preempt_disable(). preempt_disable() does not mask IRQs and the idle kfuncs are callable from IRQ-enabled contexts, so a nested invocation on the same CPU can overwrite the mask with nodes_copy() while the interrupted invocation is still iterating it, leading to a wrong node traversal and a wrong idle CPU pick. Switch to irqsave so a nested invocation can't run on the same CPU. A stack-allocated nodemask would also close the race, but that would enlarge the diff to fix a race that is already rare (per-node idle, CONFIG_NUMA and a cross-node search all at once); irqsave is the minimal fix for the context that actually triggers it. The NMI case is deliberately not addressed: there is no legitimate reason to call pick_idle from NMI and doing so poses no crash risk, so such a caller is on its own. Suggested-by: Tejun Heo Link: https://lore.kernel.org/r/d84b31727f04e1ed0d40042ba1c09e61@kernel.org Signed-off-by: Wanwu Li Signed-off-by: Tejun Heo --- kernel/sched/ext/idle.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c index d2973fb3af6d..93e2e0b2d1f8 100644 --- a/kernel/sched/ext/idle.c +++ b/kernel/sched/ext/idle.c @@ -153,7 +153,18 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i nodemask_t *unvisited; s32 cpu = -EBUSY; - preempt_disable(); + /* + * @per_cpu_unvisited is per-CPU scratch and the idle kfuncs can be + * called from IRQ-enabled contexts, so mask IRQs to keep a nested + * invocation from clobbering the mask an outer invocation is still + * iterating. + * + * NMI nesting is not handled: there is no legitimate reason to call + * pick_idle from NMI and doing so poses no crash risk, so such a + * caller is on its own. + */ + guard(irqsave)(); + unvisited = this_cpu_ptr(&per_cpu_unvisited); /* @@ -183,7 +194,6 @@ static s32 pick_idle_cpu_from_online_nodes(const struct cpumask *cpus_allowed, i if (cpu >= 0) break; } - preempt_enable(); return cpu; } From f6365b34c577756c941a716528864307d883d9dd Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Tue, 8 Sep 2026 21:47:08 +0800 Subject: [PATCH 08/69] tools/sched_ext: Drop the no-op -rdynamic from CFLAGS Because CFLAGS only applies to compilation ($(CC) -c), the link-time flag -rdynamic is inert during the build. Schedulers do not rely on exported dynamic symbols, and binaries remain byte-identical without it. Under -Werror, Clang treats this unused argument as a build error: $ make CC=clang-21 clang-21: error: argument unused during compilation: '-rdynamic' [-Werror,-Wunused-command-line-argument] Drop -rdynamic from CFLAGS to ensure clean builds across Clang versions. Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- tools/sched_ext/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile index 21554f089692..70d2503b6b33 100644 --- a/tools/sched_ext/Makefile +++ b/tools/sched_ext/Makefile @@ -89,7 +89,7 @@ ifneq ($(wildcard $(GENHDR)),) GENFLAGS := -DHAVE_GENHDR endif -CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ +CFLAGS += -g -O2 -pthread -Wall -Werror $(GENFLAGS) \ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include From 117c658adb0fb9276966a86fd4c2ed112640a0ee Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 12:19:42 +0530 Subject: [PATCH 09/69] sched_ext: Fix typo "upto" in comments Correct "upto" to "up to", reported by scripts/checkpatch.pl using the misspelling list in scripts/spelling.txt. Only touches comments, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 387953b92668..4080902a528c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5849,7 +5849,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) /* * We don't want CPUs to have more than $nr_donor_target tasks and - * balancing to fill donee CPUs upto $nr_target. Once targets are + * balancing to fill donee CPUs up to $nr_target. Once targets are * calculated, find the donee CPUs. */ nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); @@ -8825,7 +8825,7 @@ __bpf_kfunc_start_defs(); * task is inserted. * * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id - * and this function can be called upto ops.dispatch_max_batch times to insert + * and this function can be called up to ops.dispatch_max_batch times to insert * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the * remaining slots. scx_bpf_dsq_move_to_local() flushes the batch and resets the * counter. From 97c50e21e4ddfc91185d548c29c33ba631acbdc8 Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 12:19:43 +0530 Subject: [PATCH 10/69] tools/sched_ext: Fix repeated word 'be' in comment Drop the second 'be', reported by checkpatch.pl as a possible repeated word. Only touches a comment, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Signed-off-by: Tejun Heo --- tools/sched_ext/include/scx/common.bpf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 76f5e025e107..9660b43d612c 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -306,7 +306,7 @@ BPF_PROG(name, ##args) * Similar to MEMBER_VPTR() but is intended for use with arrays where the * element count needs to be explicit. * It can be used in cases where a global array is defined with an initial - * size but is intended to be be resized before loading the BPF program. + * size but is intended to be resized before loading the BPF program. * Without this version of the macro, MEMBER_VPTR() will use the compile time * size of the array to compute the max, which will result in rejection by * the verifier. From 7036264dcd46ab641768ef97ab74f91bce5701c8 Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Sun, 6 Sep 2026 21:54:55 +0800 Subject: [PATCH 11/69] selftests/sched_ext: Fail interrupted test runs SIGINT and SIGTERM set exit_req, causing the runner to stop before executing the remaining tests. However, the exit status only reflects the number of failed tests, so an interrupted run returns success when none of the completed tests failed. Include exit_req in the failure condition so callers can distinguish an interrupted run from a successful run. Keep the result counts limited to the tests that actually ran. Tested the actual runner under GDB, delivering each signal before the first test and after the example test passed. The original runner returns 0 in all four cases; the fixed runner returns 1. Normal -h, -l and -t example invocations still return 0. Fixes: 9d851afa4826 ("selftests/sched_ext: Abort test loop on signal") Assisted-by: LLM Signed-off-by: Tianyi Chen Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/runner.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/sched_ext/runner.c b/tools/testing/selftests/sched_ext/runner.c index c264807caa91..57dda75384f1 100644 --- a/tools/testing/selftests/sched_ext/runner.c +++ b/tools/testing/selftests/sched_ext/runner.c @@ -243,7 +243,7 @@ int main(int argc, char **argv) printf(" - %s\n", failed_tests[i]); } - return failed > 0 ? 1 : 0; + return failed > 0 || exit_req ? 1 : 0; } void scx_test_register(struct scx_test *test) From 49c64ffb2fd996af4c82007196c61e17d74cb1ab Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Sun, 6 Sep 2026 21:55:22 +0800 Subject: [PATCH 12/69] selftests/sched_ext: Handle CPU hotplug write failures toggle_online_status() logs failed CPU state changes but discards the write result. The hotplug tests can consequently wait indefinitely for a scheduler exit that the failed operation never triggered. Return the write result and stop both hotplug tests when a required CPU state change fails. Release the acquired scheduler resources on these paths and let the existing cleanup callback retry restoring CPU1. Also fail the test when its normal CPU restoration fails. Tested in a two-vCPU VM running a kernel built from the same source as the selftests. Injecting EIO with strace into each of the ten CPU state writes makes the fixed test return 1, with CPU1 online and sched_ext disabled after cleanup. The original test does not finish within three seconds when either of the first two writes fails. Normal hotplug tests pass before and after the change. Making the online file read-only also produces a failure without hanging. Fixes: a5db7817af78 ("sched_ext: Add selftests") Assisted-by: LLM Signed-off-by: Tianyi Chen Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/hotplug.c | 34 ++++++++++++++------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/sched_ext/hotplug.c b/tools/testing/selftests/sched_ext/hotplug.c index 0cfbb111a2d0..10b8d42bd89b 100644 --- a/tools/testing/selftests/sched_ext/hotplug.c +++ b/tools/testing/selftests/sched_ext/hotplug.c @@ -21,15 +21,17 @@ static bool is_cpu_online(void) return file_read_long(online_path) > 0; } -static void toggle_online_status(bool online) +static int toggle_online_status(bool online) { long val = online ? 1 : 0; int ret; ret = file_write_long(online_path, val); if (ret != 0) - fprintf(stderr, "Failed to bring CPU %s (%s)", + fprintf(stderr, "Failed to bring CPU %s (%s)\n", online ? "online" : "offline", strerror(errno)); + + return ret; } static enum scx_test_status setup(void **ctx) @@ -44,6 +46,7 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) { struct hotplug *skel; struct bpf_link *link; + enum scx_test_status status = SCX_TEST_FAIL; long kind, code; SCX_ASSERT(is_cpu_online()); @@ -54,8 +57,8 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) SCX_FAIL_IF(hotplug__load(skel), "Failed to load skel"); /* Testing the offline -> online path, so go offline before starting */ - if (onlining) - toggle_online_status(0); + if (onlining && toggle_online_status(0)) + goto out_destroy_skel; if (cbs_defined) { kind = SCX_KIND_VAL(SCX_EXIT_UNREG_BPF); @@ -79,7 +82,8 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) return SCX_TEST_FAIL; } - toggle_online_status(onlining ? 1 : 0); + if (toggle_online_status(onlining ? 1 : 0)) + goto out_destroy_link; while (!UEI_EXITED(skel, uei)) sched_yield(); @@ -87,20 +91,23 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) SCX_EQ(skel->data->uei.kind, kind); SCX_EQ(UEI_REPORT(skel, uei), code); - if (!onlining) - toggle_online_status(1); + if (!onlining && toggle_online_status(1)) + goto out_destroy_link; + status = SCX_TEST_PASS; +out_destroy_link: bpf_link__destroy(link); +out_destroy_skel: hotplug__destroy(skel); - return SCX_TEST_PASS; + return status; } static enum scx_test_status test_hotplug_attach(void) { struct hotplug *skel; struct bpf_link *link; - enum scx_test_status status = SCX_TEST_PASS; + enum scx_test_status status = SCX_TEST_FAIL; long kind, code; SCX_ASSERT(is_cpu_online()); @@ -115,10 +122,12 @@ static enum scx_test_status test_hotplug_attach(void) * Take the CPU offline to increment the global hotplug seq, which * should cause attach to fail due to us setting the hotplug seq above */ - toggle_online_status(0); + if (toggle_online_status(0)) + goto out_destroy_skel; link = bpf_map__attach_struct_ops(skel->maps.hotplug_nocb_ops); - toggle_online_status(1); + if (toggle_online_status(1)) + goto out_destroy_link; SCX_ASSERT(link); while (!UEI_EXITED(skel, uei)) @@ -130,7 +139,10 @@ static enum scx_test_status test_hotplug_attach(void) SCX_EQ(skel->data->uei.kind, kind); SCX_EQ(UEI_REPORT(skel, uei), code); + status = SCX_TEST_PASS; +out_destroy_link: bpf_link__destroy(link); +out_destroy_skel: hotplug__destroy(skel); return status; From 570355c81dbf3bc88e45e3ce5b3f5938cfda11bb Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Sun, 6 Sep 2026 22:40:44 +0800 Subject: [PATCH 13/69] selftests/sched_ext: Cover duplicate DSQ creation and ID reuse The create_dsq test creates and destroys queues, but does not check the -EEXIST contract for duplicate creation or whether the ID remains accessible after the failed operation. Extend the existing lifecycle loop to reject duplicate creation and check that the empty queue is still accessible. After destruction, require -ENOENT from the queue lookup, then recreate and destroy the same ID. Check that all 1024 iterations complete. This exercises empty queues during initialization, without concurrent enqueueing or destruction of a nonempty queue. The create_dsq test passes on a two-CPU matching-kernel VM. Link: https://lore.kernel.org/all/Z-OZ7tJWhRZbUk1l@gpd3/ Assisted-by: LLM Signed-off-by: Tianyi Chen Signed-off-by: Tejun Heo --- .../selftests/sched_ext/create_dsq.bpf.c | 35 +++++++++++++++++++ .../testing/selftests/sched_ext/create_dsq.c | 4 ++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/sched_ext/create_dsq.bpf.c b/tools/testing/selftests/sched_ext/create_dsq.bpf.c index 2cfc4ffd60e2..680cc4b6d8c7 100644 --- a/tools/testing/selftests/sched_ext/create_dsq.bpf.c +++ b/tools/testing/selftests/sched_ext/create_dsq.bpf.c @@ -10,6 +10,8 @@ char _license[] SEC("license") = "GPL"; +u32 nr_lifecycle_tests; + void BPF_STRUCT_OPS(create_dsq_exit_task, struct task_struct *p, struct scx_exit_task_args *args) { @@ -43,7 +45,40 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(create_dsq_init) } bpf_for(i, 0, 1024) { + err = scx_bpf_create_dsq(i, -1); + if (err != -EEXIST) { + scx_bpf_error("Duplicate DSQ %d creation returned %d", i, err); + return -EINVAL; + } + + /* A rejected duplicate must leave the original DSQ accessible. */ + err = scx_bpf_dsq_nr_queued(i); + if (err) { + scx_bpf_error("Original DSQ %d queue count is %d", i, err); + return -EINVAL; + } + scx_bpf_destroy_dsq(i); + err = scx_bpf_dsq_nr_queued(i); + if (err != -ENOENT) { + scx_bpf_error("Destroyed DSQ %d queue count is %d", i, err); + return -EINVAL; + } + + err = scx_bpf_create_dsq(i, -1); + if (err) { + scx_bpf_error("Failed to recreate DSQ %d: %d", i, err); + return err; + } + + err = scx_bpf_dsq_nr_queued(i); + if (err) { + scx_bpf_error("Recreated DSQ %d queue count is %d", i, err); + return -EINVAL; + } + + scx_bpf_destroy_dsq(i); + nr_lifecycle_tests++; } return 0; diff --git a/tools/testing/selftests/sched_ext/create_dsq.c b/tools/testing/selftests/sched_ext/create_dsq.c index d67431f57ac6..422e6532ee71 100644 --- a/tools/testing/selftests/sched_ext/create_dsq.c +++ b/tools/testing/selftests/sched_ext/create_dsq.c @@ -37,6 +37,8 @@ static enum scx_test_status run(void *ctx) bpf_link__destroy(link); + SCX_EQ(skel->bss->nr_lifecycle_tests, 1024); + return SCX_TEST_PASS; } @@ -49,7 +51,7 @@ static void cleanup(void *ctx) struct scx_test create_dsq = { .name = "create_dsq", - .description = "Create and destroy a dsq in a loop", + .description = "Create, reject duplicates, destroy and recreate DSQs", .setup = setup, .run = run, .cleanup = cleanup, From 60204f773b3851ab8320c83c84b0bd1f67189f76 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Wed, 9 Sep 2026 23:17:22 +0800 Subject: [PATCH 14/69] selftests/sched_ext: Drop -rdynamic and stop clobbering LDFLAGS CFLAGS passes the link-time flag -rdynamic to compile steps ($(CC) -c), where it is unused. Under -Werror, Clang fails the build: $ make CC=clang-21 clang-21: error: argument unused during compilation: '-rdynamic' [-Werror,-Wunused-command-line-argument] Nothing needs the exported symbols. Separately, LDFLAGS uses '=' rather than '+=', discarding USERLDFLAGS from lib.mk. Drop -rdynamic and append LDFLAGS with '+='. This fixes builds with CC=clang-N while preserving USERLDFLAGS. v2: Drop -rdynamic instead of moving it to LDFLAGS (Tejun Heo). Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 3cfe90e0f34f..5f5dd9ab903a 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -53,7 +53,7 @@ ifneq ($(wildcard $(GENHDR)),) GENFLAGS := -DHAVE_GENHDR endif -CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ +CFLAGS += -g -O2 -pthread -Wall -Werror $(GENFLAGS) \ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include -I$(SCXTOOLSINCDIR) @@ -62,7 +62,7 @@ ifneq ($(LLVM),) CFLAGS += -Wno-unused-command-line-argument endif -LDFLAGS = -lelf -lz -lpthread -lzstd +LDFLAGS += -lelf -lz -lpthread -lzstd IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - Date: Thu, 10 Sep 2026 01:27:18 +0800 Subject: [PATCH 15/69] sched_ext: Merge adjacent ifdefs in ext.h 6a1cda143c23 ("sched_ext: Always use SMP versions in kernel/sched/ext.h") dropped the CONFIG_SMP guard from scx_update_idle(), leaving two adjacent CONFIG_SCHED_CLASS_EXT blocks. Merge the declaration and stub into the primary ifdef block. No functional change. Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.h | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index 0b7fc46aee08..0012f708a550 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -21,6 +21,13 @@ int scx_check_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); void init_sched_ext_class(void); +void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); + +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) +{ + if (scx_enabled()) + __scx_update_idle(rq, idle, do_notify); +} static inline u32 scx_cpuperf_target(s32 cpu) { @@ -55,21 +62,10 @@ static inline int scx_check_setscheduler(struct task_struct *p, int policy) { re static inline bool task_on_scx(const struct task_struct *p) { return false; } static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } static inline void init_sched_ext_class(void) {} +static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} #endif /* CONFIG_SCHED_CLASS_EXT */ -#ifdef CONFIG_SCHED_CLASS_EXT -void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); - -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) -{ - if (scx_enabled()) - __scx_update_idle(rq, idle, do_notify); -} -#else -static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) {} -#endif - #ifdef CONFIG_CGROUP_SCHED #ifdef CONFIG_EXT_GROUP_SCHED void scx_tg_init(struct task_group *tg); From 46eb991947d45f7e3bf4fc9d9e3b1c3454627400 Mon Sep 17 00:00:00 2001 From: Tianyi Chen Date: Sun, 6 Sep 2026 22:34:48 +0800 Subject: [PATCH 16/69] selftests/sched_ext: Validate select_cpu_and mask constraints The syscall test only prints the CPU selection result after checking whether BPF_PROG_TEST_RUN succeeds. Require -EBUSY for empty and affinity-disjoint custom masks, and allow a legal singleton candidate to be selected or busy. Reject unexpected errors and selections outside the custom mask or configured affinity. Check cpus_mask because migration disabling can temporarily narrow cpus_ptr under CONFIG_PREEMPT_RCU. Grow dynamically allocated affinity masks until sched_getaffinity() succeeds, retaining that allocation size for pinning and restoration. Restore affinity on all exits after pinning, and skip only the disjoint case when fewer than two CPUs are allowed. Assisted-by: LLM Signed-off-by: Tianyi Chen Signed-off-by: Tejun Heo --- .../selftests/sched_ext/allowed_cpus.bpf.c | 24 +++- .../selftests/sched_ext/allowed_cpus.c | 123 ++++++++++++++++-- 2 files changed, 132 insertions(+), 15 deletions(-) diff --git a/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c b/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c index 9dd72d0da29b..f14d7e5bef9c 100644 --- a/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c +++ b/tools/testing/selftests/sched_ext/allowed_cpus.bpf.c @@ -147,23 +147,41 @@ void BPF_STRUCT_OPS(allowed_cpus_exit, struct scx_exit_info *ei) } struct task_cpu_arg { - pid_t pid; + u64 pid; + s64 custom_cpu; }; SEC("syscall") int select_cpu_from_user(struct task_cpu_arg *input) { struct task_struct *p; - int cpu; + struct bpf_cpumask *mask; + s32 cpu; p = bpf_task_from_pid(input->pid); if (!p) return -EINVAL; + mask = bpf_cpumask_create(); + if (!mask) { + bpf_task_release(p); + return -ENOMEM; + } + + /* A negative custom_cpu leaves the custom mask empty. */ + if (input->custom_cpu >= 0) + bpf_cpumask_set_cpu(input->custom_cpu, mask); + bpf_rcu_read_lock(); - cpu = scx_bpf_select_cpu_and(p, bpf_get_smp_processor_id(), 0, p->cpus_ptr, 0); + cpu = scx_bpf_select_cpu_and(p, bpf_get_smp_processor_id(), 0, + cast_mask(mask), 0); + if (cpu >= 0 && + (!bpf_cpumask_test_cpu(cpu, cast_mask(mask)) || + !bpf_cpumask_test_cpu(cpu, &p->cpus_mask))) + cpu = -ERANGE; bpf_rcu_read_unlock(); + bpf_cpumask_release(mask); bpf_task_release(p); return cpu; diff --git a/tools/testing/selftests/sched_ext/allowed_cpus.c b/tools/testing/selftests/sched_ext/allowed_cpus.c index 093f285ab4ba..773699d120ea 100644 --- a/tools/testing/selftests/sched_ext/allowed_cpus.c +++ b/tools/testing/selftests/sched_ext/allowed_cpus.c @@ -2,7 +2,10 @@ /* * Copyright (c) 2025 Andrea Righi */ +#define _GNU_SOURCE #include +#include +#include #include #include #include @@ -23,17 +26,19 @@ static enum scx_test_status setup(void **ctx) return SCX_TEST_PASS; } -static int test_select_cpu_from_user(const struct allowed_cpus *skel) +static int test_select_cpu_from_user(const struct allowed_cpus *skel, + const char *name, int custom_cpu, + bool expect_busy) { int fd, ret; - __u64 args[1]; + __s32 cpu; + __u64 args[] = { getpid(), (__u64)(__s64)custom_cpu }; LIBBPF_OPTS(bpf_test_run_opts, attr, .ctx_in = args, .ctx_size_in = sizeof(args), ); - args[0] = getpid(); fd = bpf_program__fd(skel->progs.select_cpu_from_user); if (fd < 0) return fd; @@ -42,29 +47,123 @@ static int test_select_cpu_from_user(const struct allowed_cpus *skel) if (ret < 0) return ret; - fprintf(stderr, "%s: CPU %d\n", __func__, attr.retval); + /* test_run returns the signed BPF result through an unsigned field. */ + cpu = (__s32)attr.retval; + if ((expect_busy && cpu != -EBUSY) || + (!expect_busy && cpu != -EBUSY && cpu != custom_cpu)) { + SCX_ERR("%s: unexpected CPU selection result %d", name, cpu); + return -EINVAL; + } return 0; } +/* Grow until the mask covers the kernel's CPU range, including offline CPUs. */ +static int alloc_affinity(cpu_set_t **mask, size_t *size) +{ + int nr_cpus = CPU_SETSIZE; + cpu_set_t *cpus; + int err; + + for (;;) { + *size = CPU_ALLOC_SIZE(nr_cpus); + cpus = CPU_ALLOC(nr_cpus); + if (!cpus) + return -ENOMEM; + CPU_ZERO_S(*size, cpus); + if (!sched_getaffinity(0, *size, cpus)) { + *mask = cpus; + return nr_cpus; + } + err = errno; + CPU_FREE(cpus); + if (err != EINVAL) + return -err; + if (nr_cpus > INT_MAX / 2) + return -EOVERFLOW; + nr_cpus *= 2; + } +} + static enum scx_test_status run(void *ctx) { struct allowed_cpus *skel = ctx; - struct bpf_link *link; + enum scx_test_status status = SCX_TEST_FAIL; + cpu_set_t *original = NULL, *pinned = NULL; + bool affinity_changed = false; + size_t size; + int first = -1, second = -1, cpu, nr_cpus; + struct bpf_link *link = NULL; + + nr_cpus = alloc_affinity(&original, &size); + if (nr_cpus < 0) { + SCX_ERR("Failed to get affinity (%d)", -nr_cpus); + goto out; + } + pinned = CPU_ALLOC(nr_cpus); + if (!pinned) { + SCX_ERR("Failed to allocate affinity mask"); + goto out; + } + for (cpu = 0; cpu < nr_cpus; cpu++) { + if (!CPU_ISSET_S(cpu, size, original)) + continue; + if (first < 0) { + first = cpu; + } else { + second = cpu; + break; + } + } + if (first < 0) { + SCX_ERR("No CPU in affinity mask"); + goto out; + } link = bpf_map__attach_struct_ops(skel->maps.allowed_cpus_ops); - SCX_FAIL_IF(!link, "Failed to attach scheduler"); + if (!link) { + SCX_ERR("Failed to attach scheduler"); + goto out; + } - /* Pick an idle CPU from user-space */ - SCX_FAIL_IF(test_select_cpu_from_user(skel), "Failed to pick idle CPU"); + if (test_select_cpu_from_user(skel, "empty mask", -1, true)) + goto out; - /* Just sleeping is fine, plenty of scheduling events happening */ + /* A legal candidate may be busy; selection need not succeed. */ + if (test_select_cpu_from_user(skel, "legal candidate", first, false)) + goto out; + + if (second >= 0) { + CPU_ZERO_S(size, pinned); + CPU_SET_S(first, size, pinned); + if (sched_setaffinity(0, size, pinned)) { + SCX_ERR("Failed to pin task (%d)", errno); + goto out; + } + affinity_changed = true; + if (test_select_cpu_from_user(skel, "disjoint masks", second, true)) + goto out; + } else { + fprintf(stderr, "Skipping disjoint masks: need two allowed CPUs\n"); + } + + /* Just sleeping is fine, plenty of scheduling events happening. */ sleep(1); + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + SCX_ERR("Scheduler exited unexpectedly"); + goto out; + } + status = SCX_TEST_PASS; - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_NONE)); +out: + if (affinity_changed && sched_setaffinity(0, size, original)) { + SCX_ERR("Failed to restore affinity (%d)", errno); + status = SCX_TEST_FAIL; + } bpf_link__destroy(link); - - return SCX_TEST_PASS; + CPU_FREE(pinned); + CPU_FREE(original); + return status; } static void cleanup(void *ctx) From f8e5a4e3f3bea6b389ae8dbeafe50fdb8ab543b2 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Fri, 18 Sep 2026 19:13:27 +0200 Subject: [PATCH 17/69] sched_ext: Add lazy preemption support The fair scheduling class can request lazy rescheduling, deferring an in-kernel scheduling boundary until returning to user space or until the next scheduler tick. sched_ext only exposes immediate preemption, preventing BPF schedulers from making the same trade-off. Add SCX_ENQ_PREEMPT_LAZY and SCX_KICK_PREEMPT_LAZY. Both expire the current sched_ext task slice but request lazy rescheduling. Immediate preemption, WAIT and plain kicks take precedence when requests are combined, while a lazy enqueue to a non-local DSQ retains the head-insertion semantics of SCX_ENQ_PREEMPT. Add SCX_OPS_LAZY_RESCHED as the default slice-expiry policy for newly enabled tasks and initialize it before ops.enable(). Add scx_bpf_task_set_lazy_resched() so the owning scheduler can override the persistent policy per task from any callback while preserving sub-scheduler task ownership boundaries. Bypass continues to force immediate rescheduling. For an infinite-slice task whose tick is stopped, set the lazy reschedule request before restoring the scheduler tick dependency. This lets the dependency update's IPI serve the request on return to user space and guarantees that the next tick promotes it otherwise. Use pinned rq locking for remote kick delivery so RQCF_UPDATED reflects clock updates since lock acquisition. Accumulate kick requests independently and resolve precedence while holding the target rq lock. Reject invalid SCX_KICK_IDLE combinations and unknown kick flags, which were previously silently ignored. Tested-by: Tao Cui Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 8 + kernel/sched/ext/ext.c | 151 +++++++++++++++--- kernel/sched/ext/internal.h | 44 ++++- kernel/sched/ext/sub.c | 15 +- tools/sched_ext/include/scx/compat.bpf.h | 9 ++ .../sched_ext/include/scx/enum_defs.autogen.h | 4 + .../sched_ext/include/scx/enums.autogen.bpf.h | 9 ++ tools/sched_ext/include/scx/enums.autogen.h | 3 + .../sched_ext/include/scx/enums_abi.autogen.h | 6 +- 9 files changed, 212 insertions(+), 37 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 8de69843c215..1e1fc3312bc4 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -278,6 +278,14 @@ struct sched_ext_entity { */ bool disallow; /* reject switching into SCX */ + /* + * If set, depletion of this task's slice at the scheduler tick requests + * lazy instead of immediate rescheduling. Initialized from + * %SCX_OPS_LAZY_RESCHED immediately before ops.enable() and may be + * modified afterwards with scx_bpf_task_set_lazy_resched(). + */ + bool lazy_resched; + /* cold fields */ #ifdef CONFIG_EXT_GROUP_SCHED struct cgroup *cgrp_moving_from; diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 4080902a528c..fcba7c86fe4b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -383,11 +383,11 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) return true; /* - * %SCX_ENQ_PREEMPT clears $curr's slice if on SCX and kicks dispatch, - * so allow it to avoid spuriously triggering reenq on a combined + * The preemption flags clear $curr's slice if on SCX and kick dispatch, + * so allow them to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ - if (enq_flags & SCX_ENQ_PREEMPT) { + if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { struct task_struct *curr = rq->curr; /* @@ -1515,6 +1515,26 @@ static void call_task_dequeue(struct scx_sched *sch, struct rq *rq, p->scx.flags &= ~SCX_TASK_IN_CUSTODY; } +/* + * A task with an infinite slice may be running with its tick stopped. Lazy + * rescheduling doesn't send an IPI, so restore the tick dependency to guarantee + * that the lazy request is promoted by a real scheduler tick. Set the lazy + * request first so that the dependency update's IPI can also serve it on return + * to user space. + */ +static void scx_resched_curr_lazy(struct rq *rq) +{ + resched_curr_lazy(rq); + + if (rq->scx.flags & SCX_RQ_CAN_STOP_TICK) { + rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK; + if (rq->clock_update_flags < RQCF_UPDATED) + update_rq_clock(rq); + update_other_load_avgs(rq); + sched_update_tick_dependency(rq); + } +} + static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 enq_flags) @@ -1577,12 +1597,16 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return; - if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr && + if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && p != rq->curr && rq->curr->sched_class == &ext_sched_class) { - if (likely(scx_set_task_slice(rq->curr, 0))) - resched_curr(rq); - else + if (likely(scx_set_task_slice(rq->curr, 0))) { + if (enq_flags & SCX_ENQ_PREEMPT) + resched_curr(rq); + else + scx_resched_curr_lazy(rq); + } else { __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); + } } } @@ -1672,7 +1696,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, scx_error(sch, "DSQ ID 0x%016llx already had PRIQ-enqueued tasks", dsq->id); - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) { + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { /* new task inserted at head - use fastpath */ if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); @@ -2386,7 +2410,7 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct WARN_ON_ONCE(p->scx.holding_cpu >= 0); - if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) + if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) dsq_insert_head(dst_dsq, p); else list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list); @@ -3804,8 +3828,13 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) else if (SCX_HAS_OP(sch, tick)) SCX_CALL_OP_TASK(sch, tick, rq, curr); - if (!curr->scx.slice) - resched_curr(rq); + if (!curr->scx.slice) { + /* the slice can't be trusted while bypassing */ + if (READ_ONCE(curr->scx.lazy_resched) && !scx_bypassing(sch, cpu_of(rq))) + scx_resched_curr_lazy(rq); + else + resched_curr(rq); + } } #ifdef CONFIG_EXT_GROUP_SCHED @@ -3921,6 +3950,7 @@ static void __scx_enable_task(struct scx_sched *sch, struct task_struct *p) weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO]; p->scx.weight = sched_weight_to_cgroup(weight); + p->scx.lazy_resched = sch->ops.flags & SCX_OPS_LAZY_RESCHED; if (SCX_HAS_OP(sch, enable)) SCX_CALL_OP_TASK(sch, enable, rq, p); @@ -5371,6 +5401,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_preempt_lazy); free_cpumask_var(pcpu->cpus_to_wait); exit_dsq(scx_bypass_dsq(sch, cpu)); @@ -6883,6 +6914,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, if (!cpumask_empty(pcpu->cpus_to_preempt)) scx_dump_line(&ns, " cpus_to_preempt: %*pb", cpumask_pr_args(pcpu->cpus_to_preempt)); + if (!cpumask_empty(pcpu->cpus_to_preempt_lazy)) + scx_dump_line(&ns, " preempt_lazy : %*pb", + cpumask_pr_args(pcpu->cpus_to_preempt_lazy)); if (!cpumask_empty(pcpu->cpus_to_wait)) scx_dump_line(&ns, " cpus_to_wait : %*pb", cpumask_pr_args(pcpu->cpus_to_wait)); @@ -7200,6 +7234,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, if (!zalloc_cpumask_var_node(&pcpu->cpus_to_kick, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_kick_if_idle, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt, GFP_KERNEL, node) || + !zalloc_cpumask_var_node(&pcpu->cpus_to_preempt_lazy, GFP_KERNEL, node) || !zalloc_cpumask_var_node(&pcpu->cpus_to_wait, GFP_KERNEL, node)) { ret = -ENOMEM; goto err_free_pcpu; @@ -7335,6 +7370,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd, free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); + free_cpumask_var(pcpu->cpus_to_preempt_lazy); free_cpumask_var(pcpu->cpus_to_wait); } for_each_possible_cpu(cpu) { @@ -8459,13 +8495,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r { struct rq *rq = cpu_rq(cpu); struct scx_rq *this_scx = &this_rq->scx; + struct rq_flags rf; const struct sched_class *cur_class; bool should_wait = false; bool kickable; - unsigned long flags; + bool preempt, preempt_lazy, wait, immediate; - raw_spin_rq_lock_irqsave(rq, flags); + rq_lock_irqsave(rq, &rf); cur_class = rq->curr->sched_class; + preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt); + preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy); + wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait); + /* + * Immediate preemption, waiting and a plain kick take precedence over + * lazy preemption. The lazy request still clears the slice, so all + * accumulated requests are served. + */ + immediate = preempt || wait || cpumask_test_cpu(cpu, pcpu->cpus_to_kick); /* * During CPU hotplug, a CPU may depend on kicking itself to make @@ -8479,19 +8525,23 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r !sched_class_above(cur_class, &ext_sched_class); if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) { - if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) { + if (preempt || preempt_lazy) { if (cur_class == &ext_sched_class) { u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0); - if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) + if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps))) { __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); - else if (unlikely(!scx_set_task_slice(rq->curr, 0))) + /* degrade to a plain, immediate kick */ + immediate = true; + } else if (unlikely(!scx_set_task_slice(rq->curr, 0))) { __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); + } } cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); } - if (cpumask_test_cpu(cpu, pcpu->cpus_to_wait)) { + if (wait) { if (cur_class == &ext_sched_class) { cpumask_set_cpu(cpu, this_scx->cpus_to_sync); ksyncs[cpu] = rq->scx.kick_sync; @@ -8500,17 +8550,21 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } - resched_curr(rq); + if (immediate) + resched_curr(rq); + else + scx_resched_curr_lazy(rq); } else { /* a kickable cpu was skipped solely for the missing caps */ if (kickable) __scx_add_event(pcpu->sch, SCX_EV_SUB_KICK_DENIED, 1); cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt); + cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt_lazy); cpumask_clear_cpu(cpu, pcpu->cpus_to_wait); } scx_rq_lock_drop(rq); - raw_spin_rq_unlock_irqrestore(rq, flags); + rq_unlock_irqrestore(rq, &rf); return should_wait; } @@ -8560,7 +8614,7 @@ static void kick_cpus_irq_workfn(struct irq_work *irq_work) list_for_each_entry_safe(pcpu, tmp, &this_scx->sched_pcpus_to_kick, to_kick_node) { list_del_init(&pcpu->to_kick_node); - for_each_cpu(cpu, pcpu->cpus_to_kick) { + for_each_cpu_or(cpu, pcpu->cpus_to_kick, pcpu->cpus_to_preempt_lazy) { should_wait |= kick_one_cpu(cpu, pcpu, this_rq, ksyncs); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick); cpumask_clear_cpu(cpu, pcpu->cpus_to_kick_if_idle); @@ -9531,6 +9585,41 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, return true; } +/** + * scx_bpf_task_set_lazy_resched - Set task's tick reschedule mode + * @p: task of interest + * @lazy: whether slice expiry should request lazy rescheduling + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Choose whether depletion of @p's slice at the scheduler tick requests lazy or + * immediate rescheduling. This is a persistent per-task policy and does not + * itself clear @p's slice or request rescheduling, unlike the + * %SCX_ENQ_PREEMPT_LAZY and %SCX_KICK_PREEMPT_LAZY one-shot operations. + * + * With lazy rescheduling enabled, a task interrupted in user space still + * reschedules before returning to user space. A task interrupted in the kernel + * may continue until its next return to user space or the next scheduler tick, + * which promotes the request. If lazy preemption is disabled at runtime, the + * request behaves like an immediate reschedule. + * + * @p must be on the calling scheduler. + * + * Return %true on success, %false if @p is not on the calling scheduler. + */ +__bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy, + const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + + guard(rcu)(); + sch = scx_prog_sched(aux); + if (unlikely(!sch || !scx_task_on_sched(sch, p))) + return false; + + WRITE_ONCE(p->scx.lazy_resched, lazy); + return true; +} + void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) { struct scx_sched_pcpu *pcpu; @@ -9539,6 +9628,10 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (!scx_kf_allowed_ctx(sch)) return; + if (unlikely(flags & ~SCX_KICK_ALL_FLAGS)) { + scx_error(sch, "invalid kick flags 0x%llx", flags); + return; + } local_irq_save(irq_flags); @@ -9564,7 +9657,8 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) if (flags & SCX_KICK_IDLE) { struct rq *target_rq = cpu_rq(cpu); - if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT))) + if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY | + SCX_KICK_WAIT))) scx_error(sch, "PREEMPT/WAIT cannot be used with SCX_KICK_IDLE"); if (raw_spin_rq_trylock(target_rq)) { @@ -9578,12 +9672,17 @@ void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags) } cpumask_set_cpu(cpu, pcpu->cpus_to_kick_if_idle); } else { - cpumask_set_cpu(cpu, pcpu->cpus_to_kick); - + /* + * Accumulate requests and resolve their precedence at delivery. + */ if (flags & SCX_KICK_PREEMPT) cpumask_set_cpu(cpu, pcpu->cpus_to_preempt); + if (flags & SCX_KICK_PREEMPT_LAZY) + cpumask_set_cpu(cpu, pcpu->cpus_to_preempt_lazy); if (flags & SCX_KICK_WAIT) cpumask_set_cpu(cpu, pcpu->cpus_to_wait); + if (!(flags & SCX_KICK_PREEMPT_LAZY)) + cpumask_set_cpu(cpu, pcpu->cpus_to_kick); } if (list_empty(&pcpu->to_kick_node)) @@ -9623,8 +9722,9 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux * cid-addressed equivalent of scx_bpf_kick_cpu(). An invalid @cid aborts the * scheduler via scx_cid_to_cpu(). Caps are enforced on the delivery path: a * kick is dropped if the caller lacks baseline access on @cid, and a - * %SCX_KICK_PREEMPT degrades to a plain reschedule if the caller lacks - * %SCX_CAP_PREEMPT for a task outside its subtree. + * %SCX_KICK_PREEMPT or %SCX_KICK_PREEMPT_LAZY request degrades to a plain + * reschedule if the caller lacks %SCX_CAP_PREEMPT for a task outside its + * subtree. */ __bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux) { @@ -10691,6 +10791,7 @@ __bpf_kfunc_end_defs(); BTF_KFUNCS_START(scx_kfunc_ids_any) BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU); +BTF_ID_FLAGS(func, scx_bpf_task_set_lazy_resched, KF_IMPLICIT_ARGS | KF_RCU); BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index e1eb3a0d456c..4f170cfb7986 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -215,6 +215,21 @@ enum scx_ops_flags { */ SCX_OPS_TID_TO_TASK = 1LLU << 8, + /* + * If set, tasks default to requesting lazy rescheduling when their + * slice runs out at the tick, the way fair.c expires a slice from + * update_curr(). The default is copied to p->scx.lazy_resched + * immediately before ops.enable(), after which + * scx_bpf_task_set_lazy_resched() may override it per task. + * + * A task in user space still reschedules on the way back from the tick; + * a task in the kernel runs on to its next return to user space or to + * the next tick, which promotes the request. When lazy preemption is + * disabled at runtime, the request behaves like an immediate + * reschedule. Rescheduling while bypassing stays immediate. + */ + SCX_OPS_LAZY_RESCHED = 1LLU << 9, + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING | @@ -223,7 +238,8 @@ enum scx_ops_flags { SCX_OPS_SWITCH_PARTIAL | SCX_OPS_BUILTIN_IDLE_PER_NODE | SCX_OPS_ALWAYS_ENQ_IMMED | - SCX_OPS_TID_TO_TASK, + SCX_OPS_TID_TO_TASK | + SCX_OPS_LAZY_RESCHED, /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, @@ -1327,6 +1343,7 @@ struct scx_sched_pcpu { cpumask_var_t cpus_to_kick; cpumask_var_t cpus_to_kick_if_idle; cpumask_var_t cpus_to_preempt; + cpumask_var_t cpus_to_preempt_lazy; cpumask_var_t cpus_to_wait; struct list_head to_kick_node; @@ -1407,15 +1424,16 @@ struct scx_sched_pnode { * the allocation pattern. * * ENQ_IMMED insert an IMMED task onto the cid's local DSQ - * - kick the cid's cpu (except SCX_KICK_PREEMPT) + * - kick the cid's cpu (except SCX_KICK_PREEMPT and + * SCX_KICK_PREEMPT_LAZY) * * ENQ insert any task onto the cid's local DSQ (implies ENQ_IMMED) * * PREEMPT preempt any task running on the cid regardless of the owning * sched (implies ENQ). Preempting a task in the sched's own subtree * doesn't require any cap. - * - SCX_ENQ_PREEMPT inserts - * - SCX_KICK_PREEMPT kicks + * - SCX_ENQ_PREEMPT and SCX_ENQ_PREEMPT_LAZY insert + * - SCX_KICK_PREEMPT and SCX_KICK_PREEMPT_LAZY kick * * PERF control the cid's cpu power/perf management state, currently the * cpufreq target set through scx_bpf_cidperf_set(). Hardware @@ -1685,6 +1703,15 @@ enum scx_enq_flags { */ SCX_ENQ_PREEMPT = 1LLU << 32, + /* + * Like %SCX_ENQ_PREEMPT, but request lazy rescheduling. The current + * task's slice is still cleared immediately so that the next scheduling + * boundary observes the new ordering. %SCX_ENQ_PREEMPT takes precedence + * if both are specified. Implies %SCX_ENQ_HEAD, which is all it means + * on a non-local DSQ, as with %SCX_ENQ_PREEMPT. + */ + SCX_ENQ_PREEMPT_LAZY = 1LLU << 35, + /* * Only allowed on local DSQs. Guarantees that the task either gets * on the CPU immediately and stays on it, or gets reenqueued back @@ -1811,6 +1838,15 @@ enum scx_kick_flags { * is not on SCX. */ SCX_KICK_WAIT = 1LLU << 2, + + /* + * Like %SCX_KICK_PREEMPT, but request lazy rescheduling. If combined + * with %SCX_KICK_PREEMPT or %SCX_KICK_WAIT, rescheduling is immediate. + */ + SCX_KICK_PREEMPT_LAZY = 1LLU << 3, + + SCX_KICK_ALL_FLAGS = SCX_KICK_IDLE | SCX_KICK_PREEMPT | + SCX_KICK_WAIT | SCX_KICK_PREEMPT_LAZY, }; enum scx_tg_flags { diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 380a5653dc52..1bd1b461ff30 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -686,9 +686,10 @@ void scx_rescue_init(struct rq *rq) * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on * @p. * - * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting - * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback - * migration-disabled admission. + * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT, %SCX_ENQ_PREEMPT_LAZY and %SCX_ENQ_HEAD are + * cleared when diverting to rescue or reject. %SCX_ENQ_PREEMPT and + * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled + * admission. * * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued * to and run by its nearest non-bypassing ancestor. If root is bypassing, it @@ -709,7 +710,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * On a remote activation the scheduling sched (@asch) differs from * @p's owner (@sch). Check caps against the scheduling sched. */ - if (*enq_flags & SCX_ENQ_PREEMPT) + if (*enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) needed |= scx_caps_for_preempt(asch, rq, *enq_flags); missing = scx_missing_caps(asch, cpu_of(rq), needed); @@ -726,7 +727,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r if (unlikely(!scx_rq_online(rq) || is_migration_disabled(p) || p->migration_pending)) { __scx_add_event(sch, SCX_EV_SUB_FORCED_ADMIT, 1); - *enq_flags &= ~SCX_ENQ_PREEMPT; + *enq_flags &= ~(SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY); return &rq->scx.local_dsq; } @@ -735,8 +736,8 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r * or HEAD - a diversion has no priority and IMMED is not allowed on * non-local DSQs. Strip the enq and task flags along with the slice. */ - *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD | - SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY | + SCX_ENQ_HEAD | SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); p->scx.flags &= ~SCX_TASK_IMMED; /* the enqueuer opted for rescue instead of rejection and reenqueue */ diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 6944221f96cc..7d870331b2a6 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -403,6 +403,15 @@ static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime) p->scx.dsq_vtime = vtime; } +/* v7.4: Add scx_bpf_task_set_lazy_resched(). */ +bool scx_bpf_task_set_lazy_resched___new(struct task_struct *p, bool lazy) __ksym __weak; + +static inline void scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy) +{ + if (bpf_ksym_exists(scx_bpf_task_set_lazy_resched___new)) + scx_bpf_task_set_lazy_resched___new(p, lazy); +} + /* * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This * will eventually deprecate scx_bpf_reenqueue_local(). diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 63b6b14b19bd..92501498f6a3 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -85,6 +85,7 @@ #define HAVE_SCX_ENQ_HEAD #define HAVE_SCX_ENQ_CPU_SELECTED #define HAVE_SCX_ENQ_PREEMPT +#define HAVE_SCX_ENQ_PREEMPT_LAZY #define HAVE_SCX_ENQ_IMMED #define HAVE_SCX_ENQ_RESCUE #define HAVE_SCX_ENQ_REENQ @@ -149,6 +150,8 @@ #define HAVE_SCX_KICK_IDLE #define HAVE_SCX_KICK_PREEMPT #define HAVE_SCX_KICK_WAIT +#define HAVE_SCX_KICK_PREEMPT_LAZY +#define HAVE_SCX_KICK_ALL_FLAGS #define HAVE_SCX_OPI_BEGIN #define HAVE_SCX_OPI_NORMAL_BEGIN #define HAVE_SCX_OPI_NORMAL_END @@ -164,6 +167,7 @@ #define HAVE_SCX_OPS_BUILTIN_IDLE_PER_NODE #define HAVE_SCX_OPS_ALWAYS_ENQ_IMMED #define HAVE_SCX_OPS_TID_TO_TASK +#define HAVE_SCX_OPS_LAZY_RESCHED #define HAVE_SCX_OPS_ALL_FLAGS #define HAVE___SCX_OPS_INTERNAL_MASK #define HAVE_SCX_OPS_HAS_CPU_PREEMPT diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index 7268131010de..21e1a79d5ecd 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -112,6 +112,12 @@ const volatile u64 __SCX_KICK_PREEMPT __weak; const volatile u64 __SCX_KICK_WAIT __weak; #define SCX_KICK_WAIT __SCX_KICK_WAIT +const volatile u64 __SCX_KICK_PREEMPT_LAZY __weak; +#define SCX_KICK_PREEMPT_LAZY __SCX_KICK_PREEMPT_LAZY + +const volatile u64 __SCX_KICK_ALL_FLAGS __weak; +#define SCX_KICK_ALL_FLAGS __SCX_KICK_ALL_FLAGS + const volatile u64 __SCX_ENQ_WAKEUP __weak; #define SCX_ENQ_WAKEUP __SCX_ENQ_WAKEUP @@ -121,6 +127,9 @@ const volatile u64 __SCX_ENQ_HEAD __weak; const volatile u64 __SCX_ENQ_PREEMPT __weak; #define SCX_ENQ_PREEMPT __SCX_ENQ_PREEMPT +const volatile u64 __SCX_ENQ_PREEMPT_LAZY __weak; +#define SCX_ENQ_PREEMPT_LAZY __SCX_ENQ_PREEMPT_LAZY + const volatile u64 __SCX_ENQ_IMMED __weak; #define SCX_ENQ_IMMED __SCX_ENQ_IMMED diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index e61632654517..6e0f6a3a219a 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -41,9 +41,12 @@ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_IDLE); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_WAIT); \ + SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT_LAZY); \ + SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_ALL_FLAGS); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_WAKEUP); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \ + SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT_LAZY); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_IMMED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \ diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index d53899764f5a..f1bb6d70bf1d 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -97,6 +97,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_enq_flags", "SCX_ENQ_HEAD", 0x10000LLU }, { "scx_enq_flags", "SCX_ENQ_CPU_SELECTED", 0x100000LLU }, { "scx_enq_flags", "SCX_ENQ_PREEMPT", 0x100000000LLU }, + { "scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY", 0x800000000LLU }, { "scx_enq_flags", "SCX_ENQ_IMMED", 0x200000000LLU }, { "scx_enq_flags", "SCX_ENQ_RESCUE", 0x400000000LLU }, { "scx_enq_flags", "SCX_ENQ_REENQ", 0x10000000000LLU }, @@ -161,6 +162,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_kick_flags", "SCX_KICK_IDLE", 0x1LLU }, { "scx_kick_flags", "SCX_KICK_PREEMPT", 0x2LLU }, { "scx_kick_flags", "SCX_KICK_WAIT", 0x4LLU }, + { "scx_kick_flags", "SCX_KICK_PREEMPT_LAZY", 0x8LLU }, + { "scx_kick_flags", "SCX_KICK_ALL_FLAGS", 0xfLLU }, { "scx_opi", "SCX_OPI_BEGIN", 0x0LLU }, { "scx_opi", "SCX_OPI_NORMAL_BEGIN", 0x0LLU }, { "scx_opi", "SCX_OPI_NORMAL_END", 0x21LLU }, @@ -176,7 +179,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE", 0x40LLU }, { "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU }, { "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU }, - { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x1ffLLU }, + { "scx_ops_flags", "SCX_OPS_LAZY_RESCHED", 0x200LLU }, + { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x3ffLLU }, { "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU }, { "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU }, { "scx_ops_state", "SCX_OPSS_NONE", 0x0LLU }, From ed7d96897e5389bf896fac57c1fcb2914b84cda4 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Fri, 18 Sep 2026 19:13:28 +0200 Subject: [PATCH 18/69] selftests/sched_ext: Add lazy preemption tests Add trace-based coverage for immediate and lazy preemption. An fexit probe on __resched_curr() records the requested TIF after the current task's slice is cleared. SCX_OPS_ENQ_LAST drives the victim through ops.stopping() without relying on an unrelated wakeup. Exercise kick requests individually, in both orders and combined in one call. Verify that immediate preemption and WAIT take precedence over lazy preemption, and that combining immediate and lazy enqueue preemption follows the same rule. Also test overriding the persistent policy in both directions through scx_bpf_task_set_lazy_resched(), and add invalid kick flag coverage. Extend the NO_HZ_FULL test with infinite-slice victims. Wait until the scheduler tick is observed to stop, then verify that lazy enqueue and kick requests restart it and make forward progress. Move the controller off all NO_HZ_FULL CPUs and increase the phase and watchdog timeouts to tolerate scheduling delays in oversubscribed test VMs. Detect both dynamic and static kernel preemption modes and skip mode-sensitive assertions when the active mode is unavailable. Skip trace-based cases when their kernel BTF targets are unavailable, while allowing the independent invalid-flag cases to run. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/Makefile | 1 + tools/testing/selftests/sched_ext/kick.bpf.c | 164 ++++++ tools/testing/selftests/sched_ext/kick.c | 519 ++++++++++++++++++ tools/testing/selftests/sched_ext/kick_test.h | 29 + .../selftests/sched_ext/nohz_tick.bpf.c | 57 +- tools/testing/selftests/sched_ext/nohz_tick.c | 207 ++++++- .../selftests/sched_ext/nohz_tick_test.h | 13 + tools/testing/selftests/sched_ext/util.c | 143 +++++ tools/testing/selftests/sched_ext/util.h | 12 + 9 files changed, 1112 insertions(+), 33 deletions(-) create mode 100644 tools/testing/selftests/sched_ext/kick.bpf.c create mode 100644 tools/testing/selftests/sched_ext/kick.c create mode 100644 tools/testing/selftests/sched_ext/kick_test.h create mode 100644 tools/testing/selftests/sched_ext/nohz_tick_test.h diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 5f5dd9ab903a..c4ec9b21a4c2 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -172,6 +172,7 @@ auto-test-targets := \ exit \ hotplug \ init_enable_count \ + kick \ maximal \ maybe_null \ minimal \ diff --git a/tools/testing/selftests/sched_ext/kick.bpf.c b/tools/testing/selftests/sched_ext/kick.bpf.c new file mode 100644 index 000000000000..bd9605d1573d --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick.bpf.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + */ +#include + +#include "kick_test.h" + +char _license[] SEC("license") = "GPL"; + +const volatile u32 scenario; +const volatile s32 victim_pid; +const volatile s32 challenger_pid; +const volatile s32 target_cpu; +const volatile s32 lazy_resched_override = -1; + +u32 state; +u64 slice_before; +u64 slice_at_resched; +s32 resched_tif; +u64 nr_wait_callbacks; +bool enq_preempt_issued; + +UEI_DEFINE(uei); + +static bool is_trace_scenario(void) +{ + return scenario <= TICK_EXPIRY; +} + +void BPF_STRUCT_OPS(kick_enqueue, struct task_struct *p, u64 enq_flags) +{ + switch (scenario) { + case ENQ_BOTH: + if (p->pid == victim_pid) { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_INF, enq_flags); + } else if (p->pid == challenger_pid && state == KICK_STATE_QUEUED) { + enq_preempt_issued = true; + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, + enq_flags | SCX_ENQ_PREEMPT | + SCX_ENQ_PREEMPT_LAZY); + } else { + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); + } + goto kick_last; + case INVALID_KICK_IDLE: + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_PREEMPT_LAZY | SCX_KICK_IDLE); + break; + case INVALID_KICK_UNKNOWN: + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), 1LLU << 63); + break; + } + + scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags); +kick_last: + if (enq_flags & SCX_ENQ_LAST) + scx_bpf_kick_cpu(target_cpu, SCX_KICK_IDLE); +} + +static void set_lazy_resched_override(struct task_struct *p) +{ + if (p->pid == victim_pid && lazy_resched_override >= 0) + scx_bpf_task_set_lazy_resched(p, lazy_resched_override); +} + +void BPF_STRUCT_OPS(kick_running, struct task_struct *p) +{ + if (!is_trace_scenario() || p->pid != victim_pid) + return; + set_lazy_resched_override(p); + if (__sync_val_compare_and_swap(&state, KICK_STATE_ARMED, KICK_STATE_QUEUED) != + KICK_STATE_ARMED) + return; + + slice_before = p->scx.slice; + + switch (scenario) { + case KICK_IMMEDIATE: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + break; + case KICK_LAZY: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_THEN_IMMEDIATE: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + break; + case KICK_IMMEDIATE_THEN_LAZY: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_PLAIN_THEN_LAZY: + scx_bpf_kick_cpu(target_cpu, 0); + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_THEN_PLAIN: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY); + scx_bpf_kick_cpu(target_cpu, 0); + break; + case KICK_BOTH: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT | SCX_KICK_PREEMPT_LAZY); + break; + case KICK_LAZY_WAIT: + scx_bpf_kick_cpu(target_cpu, SCX_KICK_PREEMPT_LAZY | SCX_KICK_WAIT); + break; + case ENQ_BOTH: + /* The userspace controller wakes a competing task. */ + break; + case TICK_EXPIRY: + /* no kick: the slice runs out at the tick */ + break; + } +} + +SEC("fexit/__resched_curr") +int BPF_PROG(kick_need_resched, struct rq *rq, int tif) +{ + struct task_struct *task = BPF_CORE_READ(rq, curr); + u64 slice; + + if (!task || BPF_CORE_READ(task, pid) != victim_pid || + BPF_CORE_READ(rq, cpu) != target_cpu || state != KICK_STATE_QUEUED) + return 0; + if (scenario == ENQ_BOTH && !enq_preempt_issued) + return 0; + + slice = BPF_CORE_READ(task, scx.slice); + if (slice) + return 0; + + slice_at_resched = slice; + resched_tif = tif; + state = KICK_STATE_RESCHED; + return 0; +} + +SEC("fentry/kick_sync_wait_bal_cb") +int BPF_PROG(kick_wait_callback, struct rq *rq) +{ + if (scenario == KICK_LAZY_WAIT && BPF_CORE_READ(rq, cpu) == target_cpu) + __sync_fetch_and_add(&nr_wait_callbacks, 1); + return 0; +} + +void BPF_STRUCT_OPS(kick_stopping, struct task_struct *p, bool runnable) +{ + if (p->pid == victim_pid && state == KICK_STATE_RESCHED) + state = KICK_STATE_DONE; +} + +void BPF_STRUCT_OPS(kick_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops kick_ops = { + .enqueue = (void *)kick_enqueue, + .running = (void *)kick_running, + .stopping = (void *)kick_stopping, + .exit = (void *)kick_exit, + .flags = SCX_OPS_ENQ_LAST, + .name = "kick", +}; diff --git a/tools/testing/selftests/sched_ext/kick.c b/tools/testing/selftests/sched_ext/kick.c new file mode 100644 index 000000000000..7f01602c75b2 --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kick.bpf.skel.h" +#include "kick_test.h" +#include "scx_test.h" +#include "util.h" + +#define WAIT_LOOPS 3000 + +struct observation { + u64 slice_before; + u64 slice_at_resched; + u64 nr_wait_callbacks; + s32 resched_tif; +}; + +struct kick_ctx { + cpu_set_t original_mask; + int target_cpu; +}; + +static bool enum_supported(const char *type, const char *name) +{ + u64 value; + + return __COMPAT_read_enum(type, name, &value); +} + +static enum scx_test_status setup_controller(void **ctx_ptr) +{ + struct kick_ctx *ctx; + cpu_set_t controller_mask; + int cpu, controller_cpu = -1; + + ctx = calloc(1, sizeof(*ctx)); + SCX_FAIL_IF(!ctx, "Failed to allocate context"); + if (sched_getaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask)) { + free(ctx); + SCX_FAIL("Failed to get affinity (%d)", errno); + } + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &ctx->original_mask)) + continue; + if (controller_cpu < 0) { + controller_cpu = cpu; + } else { + ctx->target_cpu = cpu; + break; + } + } + if (cpu == CPU_SETSIZE) { + printf("SKIP: two allowed CPUs are required\n"); + free(ctx); + return SCX_TEST_SKIP; + } + + CPU_ZERO(&controller_mask); + CPU_SET(controller_cpu, &controller_mask); + if (sched_setaffinity(0, sizeof(controller_mask), &controller_mask)) { + free(ctx); + SCX_FAIL("Failed to pin controller to CPU %d (%d)", controller_cpu, errno); + } + + *ctx_ptr = ctx; + return SCX_TEST_PASS; +} + +static void cleanup_controller(void *ctx_ptr) +{ + struct kick_ctx *ctx = ctx_ptr; + + sched_setaffinity(0, sizeof(ctx->original_mask), &ctx->original_mask); + free(ctx); +} + +static bool wait_for_state(struct kick *skel, u32 wanted) +{ + int i; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (__atomic_load_n(&skel->bss->state, __ATOMIC_ACQUIRE) == wanted) + return true; + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) + return false; + usleep(1000); + } + return false; +} + +static bool wait_for_counter(struct kick *skel, const u64 *counter, u64 wanted) +{ + int i; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (__atomic_load_n(counter, __ATOMIC_ACQUIRE) >= wanted) + return true; + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) + return false; + usleep(1000); + } + return false; +} + +static enum scx_test_status trace_one(struct kick_ctx *ctx, u32 scenario, u64 ops_flags, + s32 lazy_resched_override, struct observation *obs) +{ + struct bpf_link *ops_link = NULL; + struct kick *skel = NULL; + struct scx_test_gated_worker challenger = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker victim; + enum scx_test_status ret = SCX_TEST_FAIL; + int cpu = ctx->target_cpu; + + victim = scx_test_spawn_gated_worker(cpu, false); + if (victim.pid < 0) { + SCX_ERR("Failed to spawn victim"); + return SCX_TEST_FAIL; + } + if (scenario == ENQ_BOTH) { + challenger = scx_test_spawn_gated_worker(cpu, false); + if (challenger.pid < 0) { + SCX_ERR("Failed to spawn enqueue challenger"); + goto out; + } + } + + skel = kick__open(); + if (!skel) { + SCX_ERR("Failed to open scenario %u", scenario); + goto out; + } + SCX_ENUM_INIT(skel); + skel->rodata->scenario = scenario; + skel->rodata->victim_pid = victim.pid; + skel->rodata->challenger_pid = challenger.pid; + skel->rodata->target_cpu = cpu; + skel->rodata->lazy_resched_override = lazy_resched_override; + skel->struct_ops.kick_ops->flags |= ops_flags; + if (scenario != KICK_LAZY_WAIT) + bpf_program__set_autoload(skel->progs.kick_wait_callback, false); + if (kick__load(skel)) { + SCX_ERR("Failed to load scenario %u", scenario); + goto out; + } + + bpf_map__set_autoattach(skel->maps.kick_ops, false); + if (kick__attach(skel)) { + SCX_ERR("Failed to attach __resched_curr tracer"); + goto out; + } + skel->bss->state = KICK_STATE_ARMED; + ops_link = bpf_map__attach_struct_ops(skel->maps.kick_ops); + if (!ops_link) { + SCX_ERR("Failed to attach scenario %u", scenario); + goto out; + } + if (!scx_test_start_gated_worker(&victim)) { + SCX_ERR("Failed to start victim"); + goto out; + } + if (scenario == ENQ_BOTH) { + if (!wait_for_state(skel, KICK_STATE_QUEUED)) { + SCX_ERR("Enqueue scenario did not arm"); + goto out; + } + if (!scx_test_start_gated_worker(&challenger)) { + SCX_ERR("Failed to start enqueue challenger"); + goto out; + } + } + if (!wait_for_state(skel, KICK_STATE_DONE)) { + SCX_ERR("Scenario %u stopped in state %u, exit kind %d", scenario, skel->bss->state, + skel->data->uei.kind); + goto out; + } + if (scenario == KICK_LAZY_WAIT && + !wait_for_counter(skel, &skel->bss->nr_wait_callbacks, 1)) { + SCX_ERR("WAIT callback did not run"); + goto out; + } + + obs->slice_before = skel->bss->slice_before; + obs->slice_at_resched = skel->bss->slice_at_resched; + obs->nr_wait_callbacks = skel->bss->nr_wait_callbacks; + obs->resched_tif = skel->bss->resched_tif; + ret = SCX_TEST_PASS; +out: + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); + if (ops_link) + bpf_link__destroy(ops_link); + if (skel) + kick__destroy(skel); + return ret; +} + +static bool observation_valid(const struct observation *obs) +{ + return obs->slice_before > 0 && !obs->slice_at_resched; +} + +static bool trace_target_supported(const char *name, enum bpf_attach_type attach_type) +{ + return libbpf_find_vmlinux_btf_id(name, attach_type) > 0; +} + +static enum scx_test_status setup_immediate(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT")) { + printf("SKIP: SCX_KICK_PREEMPT is not supported\n"); + return SCX_TEST_SKIP; + } + if (!trace_target_supported("__resched_curr", BPF_TRACE_FEXIT)) { + printf("SKIP: __resched_curr is not available in BTF\n"); + return SCX_TEST_SKIP; + } + return setup_controller(ctx); +} + +static enum scx_test_status setup_lazy(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY")) { + printf("SKIP: SCX_KICK_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_immediate(ctx); +} + +static enum scx_test_status setup_tick(void **ctx) +{ + if (!enum_supported("scx_ops_flags", "SCX_OPS_LAZY_RESCHED")) { + printf("SKIP: SCX_OPS_LAZY_RESCHED is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_lazy(ctx); +} + +static enum scx_test_status setup_coalesce(void **ctx) +{ + if (!enum_supported("scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY")) { + printf("SKIP: SCX_ENQ_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + if (!trace_target_supported("kick_sync_wait_bal_cb", BPF_TRACE_FENTRY)) { + printf("SKIP: kick_sync_wait_bal_cb is not available in BTF\n"); + return SCX_TEST_SKIP; + } + return setup_lazy(ctx); +} + +static enum scx_test_status setup_invalid(void **ctx) +{ + if (!enum_supported("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY")) { + printf("SKIP: SCX_KICK_PREEMPT_LAZY is not supported\n"); + return SCX_TEST_SKIP; + } + return setup_controller(ctx); +} + +static enum scx_test_status run_immediate(void *ctx) +{ + struct observation obs; + enum scx_test_status status; + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &obs); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&obs)); + return SCX_TEST_PASS; +} + +static enum scx_test_status run_lazy(void *ctx) +{ + struct observation immediate, lazy; + enum scx_test_status status; + int lazy_mode = scx_test_preempt_lazy_mode(); + + if (lazy_mode < 0) { + printf("SKIP: running kernel preemption mode is unavailable\n"); + return SCX_TEST_SKIP; + } + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, KICK_LAZY, 0, -1, &lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy)); + + if (lazy_mode > 0) + SCX_FAIL_IF(lazy.resched_tif == immediate.resched_tif, + "Lazy mode used immediate TIF %d", lazy.resched_tif); + else if (!lazy_mode) + SCX_EQ(lazy.resched_tif, immediate.resched_tif); + + return SCX_TEST_PASS; +} + +static enum scx_test_status run_coalesce(void *ctx) +{ + struct observation immediate, lazy_first, immediate_first; + struct observation plain_first, lazy_then_plain; + struct observation both, lazy_wait, enq_both; + enum scx_test_status status; + + if (scx_test_preempt_lazy_mode() != 1) { + printf("SKIP: kick coalescing requires active lazy preemption\n"); + return SCX_TEST_SKIP; + } + + status = trace_one(ctx, KICK_IMMEDIATE, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, KICK_LAZY_THEN_IMMEDIATE, 0, -1, &lazy_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_first)); + status = trace_one(ctx, KICK_IMMEDIATE_THEN_LAZY, 0, -1, &immediate_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate_first)); + SCX_EQ(lazy_first.resched_tif, immediate.resched_tif); + SCX_EQ(immediate_first.resched_tif, immediate.resched_tif); + + /* + * A plain kick in the same batch doesn't clear the slice by itself. The + * lazy preemption must still expire it, and the plain kick must still + * reschedule immediately, whichever came first. + */ + status = trace_one(ctx, KICK_PLAIN_THEN_LAZY, 0, -1, &plain_first); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&plain_first)); + status = trace_one(ctx, KICK_LAZY_THEN_PLAIN, 0, -1, &lazy_then_plain); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_then_plain)); + SCX_EQ(plain_first.resched_tif, immediate.resched_tif); + SCX_EQ(lazy_then_plain.resched_tif, immediate.resched_tif); + + /* Immediate kick and WAIT both take precedence over lazy preemption. */ + status = trace_one(ctx, KICK_BOTH, 0, -1, &both); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&both)); + status = trace_one(ctx, KICK_LAZY_WAIT, 0, -1, &lazy_wait); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy_wait)); + SCX_EQ(both.resched_tif, immediate.resched_tif); + SCX_EQ(lazy_wait.resched_tif, immediate.resched_tif); + SCX_GT(lazy_wait.nr_wait_callbacks, 0); + + /* Immediate enqueue preemption likewise takes precedence over lazy. */ + status = trace_one(ctx, ENQ_BOTH, 0, -1, &enq_both); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&enq_both)); + SCX_EQ(enq_both.resched_tif, immediate.resched_tif); + return SCX_TEST_PASS; +} + +/* + * A slice running out at the tick reschedules immediately by default and lazily + * with SCX_OPS_LAZY_RESCHED, the way fair.c expires a slice. + */ +static enum scx_test_status run_tick(void *ctx) +{ + struct observation immediate, lazy, force_lazy, force_immediate; + enum scx_test_status status; + u64 lazy_flag; + int lazy_mode = scx_test_preempt_lazy_mode(); + bool found; + + if (lazy_mode < 0) { + printf("SKIP: running kernel preemption mode is unavailable\n"); + return SCX_TEST_SKIP; + } + + found = __COMPAT_read_enum("scx_ops_flags", "SCX_OPS_LAZY_RESCHED", &lazy_flag); + SCX_ASSERT(found); + status = trace_one(ctx, TICK_EXPIRY, 0, -1, &immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&immediate)); + status = trace_one(ctx, TICK_EXPIRY, lazy_flag, -1, &lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&lazy)); + if (lazy_mode > 0) { + SCX_FAIL_IF(lazy.resched_tif == immediate.resched_tif, + "Lazy slice expiry used immediate TIF %d", lazy.resched_tif); + + status = trace_one(ctx, TICK_EXPIRY, 0, 1, &force_lazy); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&force_lazy)); + status = trace_one(ctx, TICK_EXPIRY, lazy_flag, 0, &force_immediate); + SCX_EQ(status, SCX_TEST_PASS); + SCX_ASSERT(observation_valid(&force_immediate)); + SCX_EQ(force_lazy.resched_tif, lazy.resched_tif); + SCX_EQ(force_immediate.resched_tif, immediate.resched_tif); + } else { + SCX_EQ(lazy.resched_tif, immediate.resched_tif); + } + + return SCX_TEST_PASS; +} + +static enum scx_test_status invalid_one(struct kick_ctx *ctx, u32 scenario) +{ + struct bpf_link *ops_link = NULL; + struct kick *skel = NULL; + struct scx_test_gated_worker victim; + enum scx_test_status ret = SCX_TEST_FAIL; + int cpu = ctx->target_cpu; + int i; + + victim = scx_test_spawn_gated_worker(cpu, false); + if (victim.pid < 0) + return SCX_TEST_FAIL; + + skel = kick__open(); + if (!skel) + goto out; + SCX_ENUM_INIT(skel); + skel->rodata->scenario = scenario; + bpf_program__set_autoload(skel->progs.kick_need_resched, false); + bpf_program__set_autoload(skel->progs.kick_wait_callback, false); + if (kick__load(skel)) + goto out; + ops_link = bpf_map__attach_struct_ops(skel->maps.kick_ops); + if (!ops_link || !scx_test_start_gated_worker(&victim)) + goto out; + + for (i = 0; i < WAIT_LOOPS; i++) { + if (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_ERROR)) { + ret = SCX_TEST_PASS; + break; + } + usleep(1000); + } +out: + scx_test_stop_gated_worker(&victim); + if (ops_link) + bpf_link__destroy(ops_link); + if (skel) + kick__destroy(skel); + return ret; +} + +static enum scx_test_status run_invalid(void *ctx) +{ + enum scx_test_status status; + u32 scenario; + + for (scenario = INVALID_KICK_IDLE; + scenario <= INVALID_KICK_UNKNOWN; scenario++) { + status = invalid_one(ctx, scenario); + SCX_EQ(status, SCX_TEST_PASS); + } + return SCX_TEST_PASS; +} + +static struct scx_test kick_immediate = { + .name = "kick_immediate", + .description = "Trace immediate kick slice expiration and rescheduling", + .setup = setup_immediate, + .run = run_immediate, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_lazy = { + .name = "kick_lazy", + .description = "Trace lazy kick slice expiration and rescheduling", + .setup = setup_lazy, + .run = run_lazy, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_coalesce = { + .name = "kick_coalesce", + .description = "Verify lazy preemption coalesces with immediate requests", + .setup = setup_coalesce, + .run = run_coalesce, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_tick = { + .name = "kick_tick", + .description = "Trace slice expiry at the tick, immediate and lazy", + .setup = setup_tick, + .run = run_tick, + .cleanup = cleanup_controller, +}; + +static struct scx_test kick_invalid = { + .name = "kick_invalid", + .description = "Verify invalid kick flag combinations fail", + .setup = setup_invalid, + .run = run_invalid, + .cleanup = cleanup_controller, +}; + +__attribute__((constructor)) +static void register_kick_tests(void) +{ + scx_test_register(&kick_immediate); + scx_test_register(&kick_lazy); + scx_test_register(&kick_coalesce); + scx_test_register(&kick_tick); + scx_test_register(&kick_invalid); +} diff --git a/tools/testing/selftests/sched_ext/kick_test.h b/tools/testing/selftests/sched_ext/kick_test.h new file mode 100644 index 000000000000..7d3c608c6b63 --- /dev/null +++ b/tools/testing/selftests/sched_ext/kick_test.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __KICK_TEST_H__ +#define __KICK_TEST_H__ + +enum kick_scenario { + KICK_IMMEDIATE, + KICK_LAZY, + KICK_LAZY_THEN_IMMEDIATE, + KICK_IMMEDIATE_THEN_LAZY, + KICK_PLAIN_THEN_LAZY, + KICK_LAZY_THEN_PLAIN, + KICK_BOTH, + KICK_LAZY_WAIT, + ENQ_BOTH, + TICK_EXPIRY, + INVALID_KICK_IDLE, + INVALID_KICK_UNKNOWN, +}; + +enum kick_state { + KICK_STATE_IDLE, + KICK_STATE_ARMED, + KICK_STATE_QUEUED, + KICK_STATE_RESCHED, + KICK_STATE_DONE, +}; + +#endif /* __KICK_TEST_H__ */ diff --git a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c index 6998c5dd6bcb..5fec5673fbf2 100644 --- a/tools/testing/selftests/sched_ext/nohz_tick.bpf.c +++ b/tools/testing/selftests/sched_ext/nohz_tick.bpf.c @@ -6,13 +6,22 @@ */ #include +#include "nohz_tick_test.h" + char _license[] SEC("license") = "GPL"; const volatile s32 test_cpu; -bool finite_phase; +u32 phase; +s32 victim_pid; +s32 challenger_pid; +s32 trigger_pid; u64 nr_inf_running; u64 nr_finite_running; u64 nr_finite_ticks; +u64 nr_lazy_victim_running; +u64 nr_lazy_enq_running; +u64 nr_lazy_kick_running; +u64 nr_lazy_ticks; UEI_DEFINE(uei); @@ -24,9 +33,31 @@ s32 BPF_STRUCT_OPS(nohz_tick_select_cpu, struct task_struct *p, s32 prev_cpu, void BPF_STRUCT_OPS(nohz_tick_enqueue, struct task_struct *p, u64 enq_flags) { - u64 slice = finite_phase ? 1000000ULL : SCX_SLICE_INF; + u64 slice; + u64 dsq_id = SCX_DSQ_GLOBAL; - scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, slice, enq_flags); + switch (phase) { + case NOHZ_PHASE_INF: + slice = SCX_SLICE_INF; + break; + case NOHZ_PHASE_FINITE: + slice = 1000000ULL; + break; + case NOHZ_PHASE_LAZY_ENQ: + case NOHZ_PHASE_LAZY_KICK: + dsq_id = SCX_DSQ_LOCAL; + slice = p->pid == victim_pid ? SCX_SLICE_INF : SCX_SLICE_DFL; + if (phase == NOHZ_PHASE_LAZY_ENQ && p->pid == challenger_pid) + enq_flags |= SCX_ENQ_PREEMPT_LAZY; + break; + default: + slice = SCX_SLICE_DFL; + break; + } + + scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags); + if (phase == NOHZ_PHASE_LAZY_KICK && p->pid == trigger_pid) + scx_bpf_kick_cpu(test_cpu, SCX_KICK_PREEMPT_LAZY); if (enq_flags & SCX_ENQ_LAST) scx_bpf_kick_cpu(test_cpu, SCX_KICK_IDLE); } @@ -36,16 +67,28 @@ void BPF_STRUCT_OPS(nohz_tick_running, struct task_struct *p) if (bpf_get_smp_processor_id() != test_cpu) return; - if (finite_phase) + if (phase == NOHZ_PHASE_FINITE) __sync_fetch_and_add(&nr_finite_running, 1); - else + else if (phase == NOHZ_PHASE_INF) __sync_fetch_and_add(&nr_inf_running, 1); + else if (p->pid == victim_pid) + __sync_fetch_and_add(&nr_lazy_victim_running, 1); + else if (phase == NOHZ_PHASE_LAZY_ENQ && p->pid == challenger_pid) + __sync_fetch_and_add(&nr_lazy_enq_running, 1); + else if (phase == NOHZ_PHASE_LAZY_KICK && p->pid == challenger_pid) + __sync_fetch_and_add(&nr_lazy_kick_running, 1); } void BPF_STRUCT_OPS(nohz_tick_tick, struct task_struct *p) { - if (bpf_get_smp_processor_id() == test_cpu && finite_phase) + if (bpf_get_smp_processor_id() != test_cpu) + return; + + if (phase == NOHZ_PHASE_FINITE) __sync_fetch_and_add(&nr_finite_ticks, 1); + else if ((phase == NOHZ_PHASE_LAZY_ENQ || + phase == NOHZ_PHASE_LAZY_KICK) && p->pid == victim_pid) + __sync_fetch_and_add(&nr_lazy_ticks, 1); } void BPF_STRUCT_OPS(nohz_tick_exit, struct scx_exit_info *ei) @@ -61,5 +104,5 @@ struct sched_ext_ops nohz_tick_ops = { .tick = (void *)nohz_tick_tick, .exit = (void *)nohz_tick_exit, .name = "nohz_tick", - .timeout_ms = 1000U, + .timeout_ms = 5000U, }; diff --git a/tools/testing/selftests/sched_ext/nohz_tick.c b/tools/testing/selftests/sched_ext/nohz_tick.c index 028f54391c2c..df40976b9166 100644 --- a/tools/testing/selftests/sched_ext/nohz_tick.c +++ b/tools/testing/selftests/sched_ext/nohz_tick.c @@ -21,19 +21,24 @@ #include #include "nohz_tick.bpf.skel.h" +#include "nohz_tick_test.h" #include "scx_test.h" +#include "util.h" #ifndef SCHED_EXT #define SCHED_EXT 7 #endif #define MIN_FINITE_TICKS 3 -#define PHASE_TIMEOUT_MS 1000 +#define PHASE_TIMEOUT_MS 5000 +#define TICK_STOP_STABLE_MS 100 struct nohz_tick_ctx { struct nohz_tick *skel; cpu_set_t original_mask; int test_cpu; + int housekeeping_cpu; + bool test_lazy; }; static int first_allowed_cpu(const cpu_set_t *mask, int first, int last) @@ -47,20 +52,21 @@ static int first_allowed_cpu(const cpu_set_t *mask, int first, int last) return -1; } -static int find_nohz_full_cpu(const cpu_set_t *allowed) +static int read_nohz_full_mask(cpu_set_t *mask) { char buf[4096], *cur, *end; FILE *file; + int ret = 0; file = fopen("/sys/devices/system/cpu/nohz_full", "r"); if (!file) - return -1; + return -errno; if (!fgets(buf, sizeof(buf), file)) { - fclose(file); - return -1; + ret = ferror(file) ? -errno : -EINVAL; + goto out; } - fclose(file); + CPU_ZERO(mask); cur = buf; while (*cur) { long first, last; @@ -73,25 +79,30 @@ static int find_nohz_full_cpu(const cpu_set_t *allowed) errno = 0; first = strtol(cur, &end, 10); - if (errno || end == cur || first < 0 || first >= CPU_SETSIZE) - return -1; + if (errno || end == cur || first < 0 || first >= CPU_SETSIZE) { + ret = -EINVAL; + goto out; + } cur = end; last = first; if (*cur == '-') { cur++; errno = 0; last = strtol(cur, &end, 10); - if (errno || end == cur || last < first) - return -1; + if (errno || end == cur || last < first) { + ret = -EINVAL; + goto out; + } cur = end; } - cpu = first_allowed_cpu(allowed, first, last); - if (cpu >= 0) - return cpu; + for (cpu = first; cpu <= last && cpu < CPU_SETSIZE; cpu++) + CPU_SET(cpu, mask); } - return -1; +out: + fclose(file); + return ret; } static pid_t start_worker(int cpu) @@ -146,24 +157,53 @@ static int pause_worker(pid_t pid) return 0; } -static bool wait_for_counter(const u64 *counter, u64 value, int timeout_ms) +static bool wait_for_counter(struct nohz_tick *skel, const u64 *counter, u64 value, + int timeout_ms) { int elapsed; for (elapsed = 0; elapsed < timeout_ms; elapsed++) { if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value) return true; + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) + return false; usleep(1000); } return false; } +static bool wait_for_tick_stop(struct nohz_tick *skel, const u64 *counter, int timeout_ms) +{ + u64 prev = __atomic_load_n(counter, __ATOMIC_RELAXED); + int elapsed, stable = 0; + + for (elapsed = 0; elapsed < timeout_ms; elapsed++) { + u64 curr; + + usleep(1000); + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) + return false; + curr = __atomic_load_n(counter, __ATOMIC_RELAXED); + if (curr == prev) { + if (++stable >= TICK_STOP_STABLE_MS) + return true; + } else { + prev = curr; + stable = 0; + } + } + + return false; +} + static enum scx_test_status setup(void **ctx_ptr) { struct nohz_tick_ctx *ctx; - cpu_set_t controller_mask; - int cpu; + cpu_set_t controller_mask, nohz_full_mask; + bool lazy_supported; + u64 enum_value, lazy_ops_flag; + int cpu, i, lazy_mode, ret; ctx = calloc(1, sizeof(*ctx)); SCX_FAIL_IF(!ctx, "Failed to allocate context"); @@ -173,15 +213,26 @@ static enum scx_test_status setup(void **ctx_ptr) SCX_FAIL("Failed to get affinity (%d)", errno); } - cpu = find_nohz_full_cpu(&ctx->original_mask); - if (cpu < 0) { + ret = read_nohz_full_mask(&nohz_full_mask); + if (ret) { + fprintf(stderr, "SKIP: failed to read NOHZ_FULL mask (%d)\n", ret); + free(ctx); + return SCX_TEST_SKIP; + } + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) + if (CPU_ISSET(cpu, &ctx->original_mask) && CPU_ISSET(cpu, &nohz_full_mask)) + break; + if (cpu == CPU_SETSIZE) { fprintf(stderr, "SKIP: no allowed NOHZ_FULL CPU\n"); free(ctx); return SCX_TEST_SKIP; } controller_mask = ctx->original_mask; - CPU_CLR(cpu, &controller_mask); + for (i = 0; i < CPU_SETSIZE; i++) + if (CPU_ISSET(i, &nohz_full_mask)) + CPU_CLR(i, &controller_mask); if (CPU_COUNT(&controller_mask) == 0) { fprintf(stderr, "SKIP: no housekeeping CPU available\n"); free(ctx); @@ -189,6 +240,21 @@ static enum scx_test_status setup(void **ctx_ptr) } ctx->test_cpu = cpu; + ctx->housekeeping_cpu = first_allowed_cpu(&controller_mask, 0, CPU_SETSIZE - 1); + lazy_supported = __COMPAT_read_enum("scx_enq_flags", "SCX_ENQ_PREEMPT_LAZY", + &enum_value) && + __COMPAT_read_enum("scx_kick_flags", "SCX_KICK_PREEMPT_LAZY", + &enum_value) && + __COMPAT_read_enum("scx_ops_flags", "SCX_OPS_LAZY_RESCHED", + &lazy_ops_flag); + lazy_mode = scx_test_preempt_lazy_mode(); + ctx->test_lazy = lazy_supported && lazy_mode > 0; + if (lazy_supported && lazy_mode < 0) + fprintf(stderr, + "SKIP: kernel preemption mode unavailable; skipping lazy NOHZ_FULL phases\n"); + else if (lazy_supported && !lazy_mode) + fprintf(stderr, + "SKIP: lazy preemption inactive; skipping lazy NOHZ_FULL phases\n"); ctx->skel = nohz_tick__open(); if (!ctx->skel) { free(ctx); @@ -199,6 +265,8 @@ static enum scx_test_status setup(void **ctx_ptr) ctx->skel->rodata->test_cpu = cpu; ctx->skel->struct_ops.nohz_tick_ops->flags |= SCX_OPS_SWITCH_PARTIAL | SCX_OPS_ENQ_LAST; + if (lazy_supported) + ctx->skel->struct_ops.nohz_tick_ops->flags |= lazy_ops_flag; if (nohz_tick__load(ctx->skel)) { nohz_tick__destroy(ctx->skel); free(ctx); @@ -220,11 +288,15 @@ static enum scx_test_status run(void *ctx_ptr) struct nohz_tick_ctx *ctx = ctx_ptr; struct nohz_tick *skel = ctx->skel; struct bpf_link *link = NULL; + struct scx_test_gated_worker victim = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker challenger = { .pid = -1, .start_fd = -1 }; + struct scx_test_gated_worker trigger = { .pid = -1, .start_fd = -1 }; enum scx_test_status status = SCX_TEST_FAIL; pid_t finite_worker = -1; pid_t inf_worker = -1; u64 finite_running; u64 finite_ticks; + u64 victim_running; int ret; link = bpf_map__attach_struct_ops(skel->maps.nohz_tick_ops); @@ -241,7 +313,7 @@ static enum scx_test_status run(void *ctx_ptr) SCX_ERR("Failed to start infinite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_inf_running, 1, + if (!wait_for_counter(skel, &skel->bss->nr_inf_running, 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Infinite-slice worker was not scheduled"); goto out; @@ -260,18 +332,18 @@ static enum scx_test_status run(void *ctx_ptr) /* * The next EXT task receives a finite slice and must restart the tick. */ - __atomic_store_n(&skel->bss->finite_phase, true, __ATOMIC_RELEASE); + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_FINITE, __ATOMIC_RELEASE); finite_worker = start_worker(ctx->test_cpu); if (finite_worker < 0) { SCX_ERR("Failed to start finite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_running, 1, + if (!wait_for_counter(skel, &skel->bss->nr_finite_running, 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Finite-slice worker was not scheduled"); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_ticks, MIN_FINITE_TICKS, + if (!wait_for_counter(skel, &skel->bss->nr_finite_ticks, MIN_FINITE_TICKS, PHASE_TIMEOUT_MS)) { SCX_ERR("Finite-slice worker received only %llu scheduler ticks", (unsigned long long)skel->bss->nr_finite_ticks); @@ -295,12 +367,12 @@ static enum scx_test_status run(void *ctx_ptr) SCX_ERR("Failed to start second finite-slice worker (%d)", errno); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_running, + if (!wait_for_counter(skel, &skel->bss->nr_finite_running, finite_running + 1, PHASE_TIMEOUT_MS)) { SCX_ERR("Second finite-slice worker was not scheduled"); goto out; } - if (!wait_for_counter(&skel->bss->nr_finite_ticks, + if (!wait_for_counter(skel, &skel->bss->nr_finite_ticks, finite_ticks + MIN_FINITE_TICKS, PHASE_TIMEOUT_MS)) { SCX_ERR("Second finite-slice worker received only %llu scheduler ticks", @@ -308,7 +380,87 @@ static enum scx_test_status run(void *ctx_ptr) finite_ticks)); goto out; } + stop_worker(finite_worker); + finite_worker = -1; + stop_worker(inf_worker); + inf_worker = -1; + if (!ctx->test_lazy) + goto check_exit; + /* + * A lazy local enqueue must restart the tick after clearing the slice + * of an infinite-slice task on a full-dynticks CPU. + */ + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_LAZY_ENQ, __ATOMIC_RELEASE); + victim = scx_test_spawn_gated_worker(ctx->test_cpu, true); + challenger = scx_test_spawn_gated_worker(ctx->test_cpu, true); + if (victim.pid < 0 || challenger.pid < 0) { + SCX_ERR("Failed to spawn lazy-enqueue workers"); + goto out; + } + skel->bss->victim_pid = victim.pid; + skel->bss->challenger_pid = challenger.pid; + if (!scx_test_start_gated_worker(&victim) || + !wait_for_counter(skel, &skel->bss->nr_lazy_victim_running, 1, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy-enqueue victim was not scheduled"); + goto out; + } + if (!wait_for_tick_stop(skel, &skel->bss->nr_lazy_ticks, PHASE_TIMEOUT_MS)) { + SCX_ERR("Tick did not stop before lazy enqueue"); + goto out; + } + + if (!scx_test_start_gated_worker(&challenger) || + !wait_for_counter(skel, &skel->bss->nr_lazy_enq_running, 1, PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy enqueue made no progress on CPU %d", ctx->test_cpu); + goto out; + } + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); + + /* Repeat with a lazy kick delivered from a housekeeping CPU. */ + __atomic_store_n(&skel->bss->phase, NOHZ_PHASE_LAZY_KICK, __ATOMIC_RELEASE); + victim = scx_test_spawn_gated_worker(ctx->test_cpu, true); + challenger = scx_test_spawn_gated_worker(ctx->test_cpu, true); + trigger = scx_test_spawn_gated_worker(ctx->housekeeping_cpu, true); + if (victim.pid < 0 || challenger.pid < 0 || trigger.pid < 0) { + SCX_ERR("Failed to spawn lazy-kick workers"); + goto out; + } + skel->bss->victim_pid = victim.pid; + skel->bss->challenger_pid = challenger.pid; + skel->bss->trigger_pid = trigger.pid; + victim_running = __atomic_load_n(&skel->bss->nr_lazy_victim_running, + __ATOMIC_RELAXED); + if (!scx_test_start_gated_worker(&victim) || + !wait_for_counter(skel, &skel->bss->nr_lazy_victim_running, victim_running + 1, + PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy-kick victim was not scheduled"); + goto out; + } + if (!scx_test_start_gated_worker(&challenger)) { + SCX_ERR("Failed to start lazy-kick challenger"); + goto out; + } + if (!wait_for_tick_stop(skel, &skel->bss->nr_lazy_ticks, PHASE_TIMEOUT_MS)) { + SCX_ERR("Tick did not stop before lazy kick"); + goto out; + } + if (__atomic_load_n(&skel->bss->nr_lazy_kick_running, __ATOMIC_RELAXED)) { + SCX_ERR("Lazy-kick challenger ran before the kick"); + goto out; + } + if (!scx_test_start_gated_worker(&trigger) || + !wait_for_counter(skel, &skel->bss->nr_lazy_kick_running, 1, PHASE_TIMEOUT_MS)) { + SCX_ERR("Lazy kick made no progress on CPU %d", ctx->test_cpu); + goto out; + } + scx_test_stop_gated_worker(&trigger); + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); + +check_exit: if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", (unsigned long long)skel->data->uei.kind, @@ -321,6 +473,9 @@ static enum scx_test_status run(void *ctx_ptr) (unsigned long long)skel->bss->nr_finite_ticks); status = SCX_TEST_PASS; out: + scx_test_stop_gated_worker(&trigger); + scx_test_stop_gated_worker(&victim); + scx_test_stop_gated_worker(&challenger); stop_worker(finite_worker); stop_worker(inf_worker); if (link) diff --git a/tools/testing/selftests/sched_ext/nohz_tick_test.h b/tools/testing/selftests/sched_ext/nohz_tick_test.h new file mode 100644 index 000000000000..b122d723cd8b --- /dev/null +++ b/tools/testing/selftests/sched_ext/nohz_tick_test.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __NOHZ_TICK_TEST_H__ +#define __NOHZ_TICK_TEST_H__ + +enum nohz_phase { + NOHZ_PHASE_INF, + NOHZ_PHASE_FINITE, + NOHZ_PHASE_LAZY_ENQ, + NOHZ_PHASE_LAZY_KICK, +}; + +#endif /* __NOHZ_TICK_TEST_H__ */ diff --git a/tools/testing/selftests/sched_ext/util.c b/tools/testing/selftests/sched_ext/util.c index 2111329ed289..c93fc5df552d 100644 --- a/tools/testing/selftests/sched_ext/util.c +++ b/tools/testing/selftests/sched_ext/util.c @@ -5,10 +5,21 @@ */ #include #include +#include +#include #include #include #include +#include +#include #include +#include + +#include "util.h" + +#ifndef SCHED_EXT +#define SCHED_EXT 7 +#endif /* Returns read len on success, or -errno on failure. */ static ssize_t read_text(const char *path, char *buf, size_t max_len) @@ -69,3 +80,135 @@ int file_write_long(const char *path, long val) return 0; } + +struct scx_test_gated_worker scx_test_spawn_gated_worker(int cpu, bool set_sched_ext) +{ + struct scx_test_gated_worker worker = { .pid = -1, .start_fd = -1 }; + int ready[2], start[2]; + pid_t parent = getpid(); + char byte = 1; + + if (pipe(ready)) + return worker; + if (pipe(start)) { + close(ready[0]); + close(ready[1]); + return worker; + } + + worker.pid = fork(); + if (!worker.pid) { + struct sched_param param = {}; + cpu_set_t mask; + + close(ready[0]); + close(start[1]); + if (prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) + _exit(1); + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + if (sched_setaffinity(0, sizeof(mask), &mask)) + _exit(1); + if (set_sched_ext && sched_setscheduler(0, SCHED_EXT, ¶m)) + _exit(1); + if (write(ready[1], &byte, 1) != 1) + _exit(1); + close(ready[1]); + if (read(start[0], &byte, 1) != 1) + _exit(1); + close(start[0]); + for (;;) + asm volatile("" ::: "memory"); + } + if (worker.pid < 0) { + close(ready[0]); + close(ready[1]); + close(start[0]); + close(start[1]); + return worker; + } + + close(ready[1]); + close(start[0]); + if (read(ready[0], &byte, 1) != 1) { + close(ready[0]); + close(start[1]); + kill(worker.pid, SIGKILL); + waitpid(worker.pid, NULL, 0); + worker.pid = -1; + return worker; + } + close(ready[0]); + worker.start_fd = start[1]; + return worker; +} + +bool scx_test_start_gated_worker(struct scx_test_gated_worker *worker) +{ + char byte = 1; + + if (write(worker->start_fd, &byte, 1) != 1) + return false; + close(worker->start_fd); + worker->start_fd = -1; + return true; +} + +void scx_test_stop_gated_worker(struct scx_test_gated_worker *worker) +{ + if (worker->start_fd >= 0) + close(worker->start_fd); + if (worker->pid > 0) { + kill(worker->pid, SIGKILL); + waitpid(worker->pid, NULL, 0); + } + worker->pid = -1; + worker->start_fd = -1; +} + +static int config_preempt_lazy_mode(void) +{ + bool dynamic = false, lazy = false, immediate = false; + char buf[128]; + gzFile file; + + file = gzopen("/proc/config.gz", "r"); + if (!file) + return -1; + + while (gzgets(file, buf, sizeof(buf))) { + if (!strcmp(buf, "CONFIG_PREEMPT_DYNAMIC=y\n")) + dynamic = true; + else if (!strcmp(buf, "CONFIG_PREEMPT_LAZY=y\n")) + lazy = true; + else if (!strcmp(buf, "CONFIG_PREEMPT_NONE=y\n") || + !strcmp(buf, "CONFIG_PREEMPT_VOLUNTARY=y\n") || + !strcmp(buf, "CONFIG_PREEMPT=y\n") || + !strcmp(buf, "CONFIG_PREEMPT_RT=y\n")) + immediate = true; + } + gzclose(file); + + if (dynamic) + return -1; + if (lazy) + return 1; + if (immediate) + return 0; + return -1; +} + +int scx_test_preempt_lazy_mode(void) +{ + char buf[128]; + + if (read_text("/sys/kernel/debug/sched/preempt", buf, sizeof(buf)) > 0) { + if (strstr(buf, "(lazy)")) + return 1; + if (strstr(buf, "(none)") || strstr(buf, "(voluntary)") || + strstr(buf, "(full)")) + return 0; + } + + return config_preempt_lazy_mode(); +} diff --git a/tools/testing/selftests/sched_ext/util.h b/tools/testing/selftests/sched_ext/util.h index 681cec04b439..aac34fc4ef71 100644 --- a/tools/testing/selftests/sched_ext/util.h +++ b/tools/testing/selftests/sched_ext/util.h @@ -7,7 +7,19 @@ #ifndef __SCX_TEST_UTIL_H__ #define __SCX_TEST_UTIL_H__ +#include +#include + +struct scx_test_gated_worker { + pid_t pid; + int start_fd; +}; + long file_read_long(const char *path); int file_write_long(const char *path, long val); +struct scx_test_gated_worker scx_test_spawn_gated_worker(int cpu, bool set_sched_ext); +bool scx_test_start_gated_worker(struct scx_test_gated_worker *worker); +void scx_test_stop_gated_worker(struct scx_test_gated_worker *worker); +int scx_test_preempt_lazy_mode(void); #endif // __SCX_TEST_UTIL_H__ From e4c5ba819d3ce8c146a3a2d68f507e1845fdaafb Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 21 Sep 2026 10:19:28 -0700 Subject: [PATCH 19/69] sched_ext: Specialize the DSQ hashtable compare rhashtable is a generic container: it does not know what a key is, so it carries the key's shape as data. dsq_hash_params declares key_len, key_offset and head_offset and nothing else, so lookups fall back to rhashtable_compare(), which reads both the offset and the length back out of ht->p at runtime: memcmp(ptr + ht->p.key_offset, arg->key, ht->p.key_len) The key is one naturally aligned u64, but neither the offset nor the length is a compile-time constant there, so the compiler cannot narrow the call and emits an out-of-line memcmp() for every element walked - two loads, a call and a length dispatch to compare eight bytes. The interpretation costs more than the comparison it is interpreting. find_user_dsq() sits on the __schedule() path, and scx_layered calls scx_bpf_dsq_nr_queued() once per layer per dispatch decision. On Meta's fleet the lookup has a significant cost and shows up in fleet wide profile. Let's supply an obj_cmpfn. dsq_hash_params is a const object passed by value into the __always_inline __rhashtable_lookup(), so params.obj_cmpfn is a compile-time constant, the ternary that selects it folds away and the callback inlines: the compare becomes a single cmp against dsq->id. Every consumer of the result only tests it against zero, so the ordering memcmp() also carries is never observed. No functional change intended. An in-kernel A/B over the DSQ ids scx_layered creates, both parameter sets compiled into the same kernel, makes the lookup 2.9x faster. Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index fcba7c86fe4b..fe7e07d72a69 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -189,10 +189,20 @@ static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc); */ static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); +static __always_inline int dsq_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) +{ + const struct scx_dispatch_q *dsq = ptr; + + BUILD_BUG_ON(sizeof_field(struct scx_dispatch_q, id) != sizeof(u64)); + + return dsq->id != *(const u64 *)arg->key; +} + static const struct rhashtable_params dsq_hash_params = { .key_len = sizeof_field(struct scx_dispatch_q, id), .key_offset = offsetof(struct scx_dispatch_q, id), .head_offset = offsetof(struct scx_dispatch_q, hash_node), + .obj_cmpfn = dsq_cmpfn, }; static LLIST_HEAD(dsqs_to_free); From daaab0f42e2eec54a8fe585b2fa3f5408a9b3585 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 21 Sep 2026 11:59:11 -0700 Subject: [PATCH 20/69] sched_ext: Specialize the TID hashtable compare scx_tid_hash_params does not provide an object comparison function, so rhashtable falls back to rhashtable_compare(). Although the key is one naturally aligned u64, the generic comparison reads the key offset and length from the table parameters at runtime and emits an out-of-line memcmp() for each element walked. scx_bpf_tid_to_task() can sit on hot scheduling paths. Supply an obj_cmpfn so the const parameters passed into the inlined rhashtable lookup specialize the comparison to a single equality test against scx->tid. The result is only tested against zero, so the ordering provided by memcmp() is not observable. No functional change intended. Suggested-by: Tejun Heo Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index fe7e07d72a69..f5887542b502 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -52,10 +52,20 @@ struct rhashtable scx_sched_hash; #endif /* see SCX_OPS_TID_TO_TASK */ +static __always_inline int scx_tid_cmpfn(struct rhashtable_compare_arg *arg, const void *ptr) +{ + const struct sched_ext_entity *scx = ptr; + + BUILD_BUG_ON(sizeof_field(struct sched_ext_entity, tid) != sizeof(u64)); + + return scx->tid != *(const u64 *)arg->key; +} + static const struct rhashtable_params scx_tid_hash_params = { .key_len = sizeof_field(struct sched_ext_entity, tid), .key_offset = offsetof(struct sched_ext_entity, tid), .head_offset = offsetof(struct sched_ext_entity, tid_hash_node), + .obj_cmpfn = scx_tid_cmpfn, .insecure_elasticity = true, /* inserted/removed under scx_tasks_lock */ }; static struct rhashtable scx_tid_hash; From b9e602f30bea2847a0c0319c5a6e0bb73d96cd7b Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Mon, 21 Sep 2026 11:59:12 -0700 Subject: [PATCH 21/69] sched_ext: Specialize the scheduler hashtable compare scx_sched_hash_params does not provide an object comparison function, so rhashtable falls back to rhashtable_compare(). Although the sub-cgroup ID key is one naturally aligned u64, the generic comparison reads the key offset and length from the table parameters at runtime and emits an out-of-line memcmp() for each element walked. Supply an obj_cmpfn so lookups specialize to a single equality test against ops.sub_cgroup_id. ext.c and sub.c are included in the same build_policy.c translation unit, so the compiler can fold the const callback into scx_find_sub_sched() even though its source is in sub.c. The result is only tested against zero, so the ordering provided by memcmp() is not observable. No functional change intended. Suggested-by: Tejun Heo Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f5887542b502..f60894dbf062 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -41,10 +41,21 @@ struct scx_sched __rcu *scx_root; LIST_HEAD(scx_sched_all); #ifdef CONFIG_EXT_SUB_SCHED +static __always_inline int scx_sched_cmpfn(struct rhashtable_compare_arg *arg, + const void *ptr) +{ + const struct scx_sched *sch = ptr; + + BUILD_BUG_ON(sizeof_field(struct scx_sched, ops.sub_cgroup_id) != sizeof(u64)); + + return sch->ops.sub_cgroup_id != *(const u64 *)arg->key; +} + const struct rhashtable_params scx_sched_hash_params = { .key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id), .key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id), .head_offset = offsetof(struct scx_sched, hash_node), + .obj_cmpfn = scx_sched_cmpfn, .insecure_elasticity = true, /* inserted under scx_sched_lock */ }; From 6b8d2bc44fbb6d232382141ae70d4ecc28a9dc36 Mon Sep 17 00:00:00 2001 From: Cheng-Yang Chou Date: Tue, 22 Sep 2026 23:50:00 +0800 Subject: [PATCH 22/69] selftests/sched_ext: Fix rt_stall runner abort rt_stall uses ksft_exit_fail() and friends, which call exit(), so a single rt_stall failure or a fork() error kills the runner and skips the remaining tests. Drop the ksft_* usage, report the handshake failure back from wait_ready(), and let run() return SCX_TEST_FAIL so the runner records the failure and moves on. Since the test now returns rather than exiting, route every path after the forks through one cleanup label, so a failure cannot leave the SCHED_FIFO task spinning on CORE_ID for the tests that follow. Also destroy the struct_ops link before checking the exit kind, so the link is released when that check fails. Fixes: be621a76341c ("selftests/sched_ext: Add test for sched_ext dl_server") Signed-off-by: Cheng-Yang Chou Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/rt_stall.c | 80 ++++++++++---------- 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/tools/testing/selftests/sched_ext/rt_stall.c b/tools/testing/selftests/sched_ext/rt_stall.c index a5041fc2e44f..a1552b38a6e8 100644 --- a/tools/testing/selftests/sched_ext/rt_stall.c +++ b/tools/testing/selftests/sched_ext/rt_stall.c @@ -17,7 +17,6 @@ #include #include "rt_stall.bpf.skel.h" #include "scx_test.h" -#include "../kselftest.h" #define CORE_ID 0 /* CPU to pin tasks to */ #define RUN_TIME 5 /* How long to run the test in seconds */ @@ -35,15 +34,17 @@ static void signal_ready(int fd) } /* Wait for a child to signal readiness via a pipe */ -static void wait_ready(int fd) +static bool wait_ready(int fd) { + bool ready; char c; - if (read(fd, &c, 1) != 1) { + ready = read(fd, &c, 1) == 1; + if (!ready) perror("read from ready pipe"); - exit(EXIT_FAILURE); - } close(fd); + + return ready; } /* Simple busy-wait function for test tasks */ @@ -151,13 +152,11 @@ static bool sched_stress_test(bool is_ext) float ext_runtime, rt_runtime, actual_ratio; int ext_pid, rt_pid; int ext_ready[2], rt_ready[2]; - - ksft_print_header(); - ksft_set_plan(1); + bool ret = false; if (pipe(ext_ready) || pipe(rt_ready)) { perror("pipe"); - ksft_exit_fail(); + return false; } /* Create and set up a EXT task */ @@ -172,7 +171,7 @@ static bool sched_stress_test(bool is_ext) exit(0); } else if (ext_pid < 0) { perror("fork task"); - ksft_exit_fail(); + return false; } /* Create an RT task */ @@ -188,7 +187,7 @@ static bool sched_stress_test(bool is_ext) exit(0); } else if (rt_pid < 0) { perror("fork for RT task"); - ksft_exit_fail(); + goto out_kill_ext; } /* @@ -199,45 +198,47 @@ static bool sched_stress_test(bool is_ext) */ close(ext_ready[1]); close(rt_ready[1]); - wait_ready(ext_ready[0]); - wait_ready(rt_ready[0]); + if (!wait_ready(ext_ready[0]) || !wait_ready(rt_ready[0])) + goto out_kill; /* Let the processes run for the specified time */ sleep(RUN_TIME); /* Get runtime for the EXT task */ ext_runtime = get_process_runtime(ext_pid); - if (ext_runtime == -1) - ksft_exit_fail_msg("Error getting runtime for %s task (PID %d)\n", - class_str, ext_pid); - ksft_print_msg("Runtime of %s task (PID %d) is %f seconds\n", - class_str, ext_pid, ext_runtime); + if (ext_runtime == -1) { + fprintf(stderr, "Failed to read %s task runtime\n", class_str); + goto out_kill; + } /* Get runtime for the RT task */ rt_runtime = get_process_runtime(rt_pid); - if (rt_runtime == -1) - ksft_exit_fail_msg("Error getting runtime for RT task (PID %d)\n", rt_pid); - ksft_print_msg("Runtime of RT task (PID %d) is %f seconds\n", rt_pid, rt_runtime); - - /* Kill the processes */ - kill(ext_pid, SIGKILL); - kill(rt_pid, SIGKILL); - waitpid(ext_pid, NULL, 0); - waitpid(rt_pid, NULL, 0); + if (rt_runtime == -1) { + fprintf(stderr, "Failed to read RT task runtime\n"); + goto out_kill; + } /* Verify that the scx task got enough runtime */ actual_ratio = ext_runtime / (ext_runtime + rt_runtime); - ksft_print_msg("%s task got %.2f%% of total runtime\n", - class_str, actual_ratio * 100); + fprintf(stderr, "%s task ran %.3fs, RT task ran %.3fs (%.2f%% of runtime)\n", + class_str, ext_runtime, rt_runtime, actual_ratio * 100); - if (actual_ratio >= expected_min_ratio) { - ksft_test_result_pass("PASS: %s task got more than %.2f%% of runtime\n", - class_str, expected_min_ratio * 100); - return true; + if (actual_ratio < expected_min_ratio) { + fprintf(stderr, "%s task got less than %.2f%% of runtime\n", + class_str, expected_min_ratio * 100); + goto out_kill; } - ksft_test_result_fail("FAIL: %s task got less than %.2f%% of runtime\n", - class_str, expected_min_ratio * 100); - return false; + + ret = true; + +out_kill: + kill(rt_pid, SIGKILL); + waitpid(rt_pid, NULL, 0); +out_kill_ext: + kill(ext_pid, SIGKILL); + waitpid(ext_pid, NULL, 0); + + return ret; } static enum scx_test_status run(void *ctx) @@ -263,14 +264,17 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.rt_stall_ops); SCX_FAIL_IF(!link, "Failed to attach scheduler"); } + res = sched_stress_test(is_ext); + if (is_ext) { - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_NONE)); + int exit_kind = skel->data->uei.kind; bpf_link__destroy(link); + SCX_EQ(exit_kind, EXIT_KIND(SCX_EXIT_NONE)); } if (!res) - ksft_exit_fail(); + return SCX_TEST_FAIL; } return SCX_TEST_PASS; From efa7bc7d719558483b14e1c44bcce11532a44581 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 23 Sep 2026 07:07:50 -0700 Subject: [PATCH 23/69] sched_ext: Avoid relocking DSQ during remote consumption A remote consume unlinks a task from a non-local DSQ before switching to the task's rq. Leaving p->scx.dsq set makes the following deactivate_task() reacquire the DSQ lock in scx_dispatch_dequeue() to clear the transfer state. Keep p->scx.dsq set while switching rq locks so that a racing dequeue can find the DSQ and cancel the transfer. After acquiring the source rq and confirming ownership, clear p->scx.dsq. The subsequent dequeue then takes its !dsq path without reacquiring the DSQ lock. Fleet profiles show that roughly 70% of the cycles attributed to scx_dispatch_dequeue() are spent in queued_spin_lock_slowpath() while taking dsq->lock. Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f60894dbf062..03c85ad0ead4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2599,7 +2599,7 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is - * still valid. %false if lost to dequeue. + * still valid and detached from @dsq. %false if lost to dequeue. */ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, struct scx_dispatch_q *dsq, @@ -2619,8 +2619,18 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, switch_rq_lock(locked_rq, src_rq); /* task_rq couldn't have changed if we're still the holding cpu */ - return likely(p->scx.holding_cpu == cpu) && - !WARN_ON_ONCE(src_rq != task_rq(p)); + if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { + /* + * Keep ->dsq set until we own @src_rq so that a racing dequeue + * can find @dsq and clear holding_cpu. Once ownership is + * confirmed, clear it under @src_rq so that deactivate_task() + * takes the !dsq path instead of retaking @dsq->lock. + */ + p->scx.dsq = NULL; + return true; + } + + return false; } static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, From 84a0d325b0b709962772e66a804677ceaca417e2 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 23 Sep 2026 14:34:44 -0700 Subject: [PATCH 24/69] sched_ext: Avoid relocking DSQ during remote DSQ moves When scx_bpf_dsq_move*() moves a task from a non-local DSQ to a remote local DSQ, move_task_between_dsqs() drops src_dsq->lock while leaving p->scx.dsq set. The following deactivate_task() then reacquires src_dsq->lock in scx_dispatch_dequeue() to unlink the task and clear p->scx.dsq. Both the source rq and src_dsq are already locked before the source DSQ lock is dropped. Call dispatch_dequeue_locked() there so that deactivate_task() takes the !dsq path and avoids the redundant lock acquisition. Suggested-by: Tejun Heo Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 03c85ad0ead4..911bb6d433ba 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2697,6 +2697,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq); raw_spin_unlock(&src_dsq->lock); } else { + dispatch_dequeue_locked(p, src_dsq); raw_spin_unlock(&src_dsq->lock); move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); } From e9b9ad3c138020dffcaddede86d91c984682806b Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Thu, 24 Sep 2026 06:05:03 -0700 Subject: [PATCH 25/69] sched_ext: Update scx_dispatch_dequeue() comments Remote consumption now reaches the !dsq branch with holding_cpu set, just like dispatch_to_local_dsq(). Clearing holding_cpu in this branch also tells dispatch_to_local_dsq() that it lost to a dequeue. The dsq-present branch races with unlink_dsq_and_switch_rq_lock(), not dispatch_to_local_dsq(). Update both comments to describe the current transfer paths and race semantics. No functional change. Suggested-by: Tejun Heo Signed-off-by: Usama Arif Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 911bb6d433ba..d01f44839da3 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1821,10 +1821,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) list_del_init(&p->scx.dsq_list.node); /* - * When dispatching directly from the BPF scheduler to a local - * DSQ, the task isn't associated with any DSQ but - * @p->scx.holding_cpu may be set under the protection of - * %SCX_OPSS_DISPATCHING. + * When dispatch_to_local_dsq() or remote consumption moves a + * task to a local DSQ, the task isn't associated with any DSQ + * but @p->scx.holding_cpu may be set. Clearing holding_cpu + * tells dispatch_to_local_dsq() that it lost to a dequeue. */ if (p->scx.holding_cpu >= 0) p->scx.holding_cpu = -1; @@ -1844,10 +1844,10 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p) scx_task_unlink_from_dsq(p, dsq); } else { /* - * We're racing against dispatch_to_local_dsq() which already - * removed @p from @dsq and set @p->scx.holding_cpu. Clear the - * holding_cpu which tells dispatch_to_local_dsq() that it lost - * the race. + * We're racing against unlink_dsq_and_switch_rq_lock(), + * which already removed @p from @dsq and set + * @p->scx.holding_cpu. Clear holding_cpu to tell + * unlink_dsq_and_switch_rq_lock() that it lost the race. */ WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node)); p->scx.holding_cpu = -1; From a87743bd9e9bef4b69d6fcce992cdaaa8edb78e3 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Thu, 24 Sep 2026 09:14:46 -0700 Subject: [PATCH 26/69] sched_ext: Place dsq_vtime next to dsq_priq scx_dispatch_enqueue() calls rb_add() for a vtime-ordered DSQ while holding dsq->lock. At each level of the descent, scx_dsq_priq_less() loads the visited task's dsq_vtime, and that comparison selects rb_left or rb_right from the same task's dsq_priq. In the x86-64 benchmark configuration, these accesses fell on different 64-byte cache lines before this change, so a cold visited task could require a second cache line fill on the dependent descent path. Move dsq_vtime immediately before dsq_priq, keeping dsq_seq and dsq_flags next to dsq_list. In the two x86-64 layouts inspected, where scx starts at offsets 776 and 840 in task_struct, the fields from dsq_list through dsq_priq occupy one cache line. The exact cache line placement depends on the containing task_struct layout and is not guaranteed for every configuration or architecture. Where they share a line, avoiding the dependent cache line fill helps reduce dsq->lock hold time during vtime insertion and therefore contention on the lock. In a 16-vCPU KVM guest, the instrumented enqueue interval was 4-6% shorter with scx_lavd and scx_layered. scx_mitosis and end-to-end workload time showed no consistent change. This only reorders fields; no scheduling behavior change is intended. Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 1e1fc3312bc4..b85676871624 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -197,9 +197,19 @@ struct sched_ext_entity { u64 ddsp_slice; u64 ddsp_vtime; struct scx_dsq_list_node dsq_list; /* dispatch order */ - struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 dsq_seq; u32 dsq_flags; /* protected by DSQ lock */ + + /* + * Used to order tasks when dispatching to the vtime-ordered priority + * queue of a dsq. This is usually set through + * scx_bpf_dsq_insert_vtime() but can also be modified directly by the + * BPF scheduler. Modifying it while a task is queued on a dsq may + * mangle the ordering and is not recommended. Kept next to @dsq_priq + * as rbtree insertion reads both on every visited node. + */ + u64 dsq_vtime; + struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 flags; /* protected by rq lock */ u32 weight; u32 reenq_cnt; /* reenqueues since last run */ @@ -225,7 +235,7 @@ struct sched_ext_entity { u64 tid; struct rhash_head tid_hash_node; /* see SCX_OPS_TID_TO_TASK */ - /* BPF scheduler modifiable fields */ + /* BPF scheduler modifiable fields, along with @dsq_vtime above */ /* * Runtime budget in nsecs - how long the task may hold its cpu. Owned @@ -240,15 +250,6 @@ struct sched_ext_entity { */ u64 slice; - /* - * Used to order tasks when dispatching to the vtime-ordered priority - * queue of a dsq. This is usually set through - * scx_bpf_dsq_insert_vtime() but can also be modified directly by the - * BPF scheduler. Modifying it while a task is queued on a dsq may - * mangle the ordering and is not recommended. - */ - u64 dsq_vtime; - /* * Out-of-band slice request from scx_bpf_task_set_slice() when the * caller does not hold the rq lock, applied under the rq lock at the From 7a919c7f86de8a0bbfd5da4eacdb67833dfe3e5a Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Thu, 24 Sep 2026 13:27:11 -0700 Subject: [PATCH 27/69] sched_ext: Test scx_has_subs() inline before calling sub-sched hooks scx_reenq_reject(), scx_process_sync_ecaps() and scx_resolve_local_dsq() return early if there are no sub-schedulers, but clang emits the full prologue before the static branch. Without sub-schedulers, we still pay for a call and for saving and restoring up to six registers, on every wakeup, dispatch and local DSQ insert. Let's move the scx_has_subs() test into inline wrappers, so that these calls go away completely. The wrappers keep lockdep_assert_rq_held(). In a pinned perf bench sched pipe test on x86-64 with clang 22, this reduced kernel instructions by 52-54 per context switch with scx_simple, scx_lavd and scx_layered. No functional change intended. Signed-off-by: Usama Arif Signed-off-by: Tejun Heo --- kernel/sched/ext/sub.c | 25 +++++++++---------------- kernel/sched/ext/sub.h | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 1bd1b461ff30..3ef4147c2d0a 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -675,7 +675,7 @@ void scx_rescue_init(struct rq *rq) } /** - * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert + * __scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert * @sch: enqueuing sub-sched * @rq: rq whose local DSQ @p targets * @p: task being inserted @@ -695,12 +695,9 @@ void scx_rescue_init(struct rq *rq) * to and run by its nearest non-bypassing ancestor. If root is bypassing, it * always holds all caps. */ -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags) +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { - if (!scx_has_subs()) - return &rq->scx.local_dsq; - s32 cid = __scx_cpu_to_cid(cpu_of(rq)); struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; u64 needed = scx_caps_for_enq(*enq_flags); @@ -783,14 +780,12 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. * Rejection can't happen for root. */ -void scx_reenq_reject(struct rq *rq) +void __scx_reenq_reject(struct rq *rq) { LIST_HEAD(tasks); struct task_struct *p, *n; - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + if (list_empty(&rq->scx.reject_dsq.list)) return; /* @@ -925,7 +920,7 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); /* - * Pairs with smp_mb() in scx_process_sync_ecaps(). Either the check + * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check * below sees the node off the list and queues it, or the in-flight sync * sees the caps[] update made before this call. */ @@ -950,7 +945,7 @@ static void discard_queued_syncs(struct rq *rq) } /** - * scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] + * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch * @@ -962,16 +957,14 @@ static void discard_queued_syncs(struct rq *rq) * learns the cid's idle state. Such a gain arms the per-rq * %SCX_RQ_SUB_IDLE_RENOTIFY gate so the next idle pick delivers it. */ -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { s32 cpu = cpu_of(rq); s32 cid, shard; struct llist_node *batch, *pos, *tmp; u64 lost_all = 0; - lockdep_assert_rq_held(rq); - - if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync))) + if (likely(llist_empty(&rq->scx.ecaps_to_sync))) return; /* diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index f7bcdfda8dd8..6a7076b9a1dd 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -28,16 +28,16 @@ bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux); void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); -void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); void scx_discard_stale_ecaps_syncs(void); -struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, - struct task_struct *p, u64 *enq_flags); +struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); -void scx_reenq_reject(struct rq *rq); +void __scx_reenq_reject(struct rq *rq); void scx_rescue_charge(struct rq *rq, s64 delta_exec); void scx_rescue_end(struct rq *rq); bool scx_rescue_keep(struct rq *rq, struct task_struct *p); @@ -72,6 +72,33 @@ static inline void scx_dec_has_subs(struct scx_sched *sch) static_branch_dec(&__scx_has_subs); } +/* hot-path hooks, gated inline so that a root-only system skips the calls */ +static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) +{ + lockdep_assert_rq_held(rq); + + if (scx_has_subs()) + __scx_process_sync_ecaps(rq, prev); +} + +static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, + struct rq *rq, + struct task_struct *p, + u64 *enq_flags) +{ + if (!scx_has_subs()) + return &rq->scx.local_dsq; + return __scx_resolve_local_dsq(sch, rq, p, enq_flags); +} + +static inline void scx_reenq_reject(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + if (scx_has_subs()) + __scx_reenq_reject(rq); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } From 5cadf7545cc6528cc41d9d8dac52faf331cbc878 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Fri, 25 Sep 2026 16:46:04 +0200 Subject: [PATCH 28/69] sched_ext: Add a CID NUMA node lookup kfunc CID-form schedulers may need the physical NUMA node for a CID, for example to allocate arena memory on that node. The CID topology's node index is not a physical NUMA node ID, and scx_bpf_cpu_node() is restricted to CPU-form schedulers. Add scx_bpf_cid_node() to return the physical NUMA node for a CID and provide a compatibility helper that returns NUMA_NO_NODE on kernels without this kfunc. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 24 ++++++++++++++++++++++++ tools/sched_ext/include/scx/common.bpf.h | 1 + tools/sched_ext/include/scx/compat.bpf.h | 5 +++++ 3 files changed, 30 insertions(+) diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 39f88deb94bc..932f640ccc2d 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -640,6 +640,29 @@ __bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux) return scx_cid_to_cpu(sch, cid); } +/** + * scx_bpf_cid_node - Return the NUMA node the given @cid belongs to + * @cid: cid to look up + * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs + * + * Return NUMA_NO_NODE if @cid is invalid or there is no scheduler. Unlike + * scx_bpf_cpu_node(), this kfunc is available only with CID support, so BPF + * schedulers can use its presence to detect NUMA-aware CID lookups. + */ +__bpf_kfunc s32 scx_bpf_cid_node(s32 cid, const struct bpf_prog_aux *aux) +{ + struct scx_sched *sch; + s32 cpu; + + guard(rcu)(); + + sch = scx_prog_sched(aux); + if (unlikely(!sch)) + return NUMA_NO_NODE; + cpu = scx_cid_to_cpu(sch, cid); + return cpu < 0 ? NUMA_NO_NODE : cpu_to_node(cpu); +} + /** * scx_bpf_cpu_to_cid - Return the cid for @cpu * @cpu: cpu to look up @@ -952,6 +975,7 @@ static const struct btf_kfunc_id_set scx_kfunc_set_init_cids = { BTF_KFUNCS_START(scx_kfunc_ids_cid) BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cid_node, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS) BTF_KFUNCS_END(scx_kfunc_ids_cid) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 9660b43d612c..41342f1bdd31 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -106,6 +106,7 @@ u64 scx_bpf_now(void) __ksym __weak; void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; +s32 scx_bpf_cid_node(s32 cid) __ksym __weak; void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out) __ksym __weak; void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 7d870331b2a6..c5c90d1d81e0 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -243,6 +243,11 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) scx_bpf_pick_any_cpu_node(cpus_allowed, node, flags) : \ scx_bpf_pick_any_cpu(cpus_allowed, flags)) +/* v7.4: Add scx_bpf_cid_node(). */ +#define __COMPAT_scx_bpf_cid_node(cid) \ + (bpf_ksym_exists(scx_bpf_cid_node) ? \ + scx_bpf_cid_node(cid) : NUMA_NO_NODE) + /* * v6.18: Add a helper to retrieve the current task running on a CPU. * From 9fa10e3bc7ec61010aa91cfca6118218e3bab773 Mon Sep 17 00:00:00 2001 From: Fuyu Zhao Date: Mon, 28 Sep 2026 11:09:55 +0800 Subject: [PATCH 29/69] tools/sched_ext: Add SCX_OPS_OPEN_OPTS for schedulers using open opts Add SCX_OPS_OPEN_OPTS() and SCX_OPS_CID_OPEN_OPTS() to allow sched_ext schedulers to pass bpf_object_open_opts when opening BPF skeletons. This allows sched_ext schedulers using open opts to retain the existing compatibility handling instead of bypassing it. Signed-off-by: Fuyu Zhao Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- tools/sched_ext/include/scx/compat.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index 7c12df45fdba..d4beb7a7c68a 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -274,23 +274,23 @@ static inline long scx_hotplug_seq(void) * - v7.1: ops.sub_attach(), ops.sub_detach(), ops.sub_cgroup_id * - v7.3: ops.rescue_bandwidth_ppt, ops.rescue_quantum_us */ -#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct) ({ \ +#define __SCX_OPS_OPEN(__ops_name, __scx_name, __ops_struct, __opts) ({ \ struct __scx_name *__oskel; \ \ SCX_BUG_ON(!__COMPAT_struct_has_field(__ops_struct, "dump"), \ __ops_struct ".dump() missing, kernel too old?"); \ \ - __oskel = __scx_name##__open(); \ + __oskel = __scx_name##__open_opts(__opts); \ SCX_BUG_ON(!__oskel, "Could not open " #__scx_name); \ __oskel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \ SCX_ENUM_INIT(__oskel); \ __oskel; \ }) -#define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \ +#define SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \ struct __scx_name *__skel; \ \ - __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops"); \ + __skel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops", __opts); \ if (__skel->struct_ops.__ops_name->cgroup_set_bandwidth && \ !__COMPAT_struct_has_field("sched_ext_ops", "cgroup_set_bandwidth")) { \ fprintf(stderr, "WARNING: kernel doesn't support ops.cgroup_set_bandwidth()\n"); \ @@ -329,12 +329,18 @@ static inline long scx_hotplug_seq(void) __skel; \ }) +#define SCX_OPS_OPEN(__ops_name, __scx_name) \ + SCX_OPS_OPEN_OPTS(__ops_name, __scx_name, 0) + /* * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates * every op the load-time fix-ups above handle, so none of them apply. */ +#define SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, __opts) \ + __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid", __opts) + #define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \ - __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid") + SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, 0) /* * Associate non-struct_ops BPF programs with the scheduler's struct_ops map so From 80b97ad2dfd8c812596ca86821cbe68f496da13d Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sun, 27 Sep 2026 15:37:19 +0200 Subject: [PATCH 30/69] sched_ext: cid: Represent clusters explicitly The sched_ext CID topology gives each core, LLC and NUMA node a contiguous CID range. Without cluster entries, schedulers cannot obtain their CID range or global index. Walk cores by cache-sharing cluster and add cluster_cid and cluster_idx to scx_cid_topo. Treat the level inclusively: a core without a wider cluster forms one of its own, while an LLC-wide cache-sharing group forms one cluster. This gives every cluster a contiguous CID range and a dense index. The walk now takes cores in cluster order rather than LLC order. Shards are still cut on core boundaries, so a shard doesn't need to contain a whole cluster; only core, cluster, LLC and node are nested, in that order. The level follows whatever the arch reports through topology_cluster_cpumask(), which on x86 is the L2 sharing mask and is populated independently of CONFIG_SCHED_CLUSTER. Tested on a 13th Gen Intel Core i7-13800H: six SMT P-cores occupy CPUs 0-11, eight E-cores occupy CPUs 12-19 in two four-core L2 modules, and all 20 CPUs share one LLC; the P-core pairs form two-CID clusters, the E-core modules form four-CID clusters, and cluster_idx is dense across all eight clusters. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 43 +++++++++++++++++++++++++++++++--------- kernel/sched/ext/cid.h | 11 +++++----- kernel/sched/ext/types.h | 28 +++++++++++++++++++++----- 3 files changed, 63 insertions(+), 19 deletions(-) diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index dc670975c5bf..9fc219257004 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -31,6 +31,7 @@ static struct scx_cid_tables *scx_cid_tables; /* used only during alloc/free */ #define SCX_CID_TOPO_NEG (struct scx_cid_topo) { \ .core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1, \ .node_cid = -1, .node_idx = -1, .shard_cid = -1, .shard_idx = -1, \ + .cluster_cid = -1, .cluster_idx = -1, \ } /* @@ -182,12 +183,14 @@ s32 scx_cid_init(struct scx_sched *sch) cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; + cpumask_var_t cluster_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL; cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL; struct scx_cid_tables *tbls; u32 next_cid = 0; s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0; + s32 next_cluster_idx = 0; s32 next_shard_idx = 0; u32 shard_size, max_cids; u32 notopo_in_shard; @@ -215,6 +218,7 @@ s32 scx_cid_init(struct scx_sched *sch) if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) || !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) || + !zalloc_cpumask_var(&cluster_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) || !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) || !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL)) @@ -256,6 +260,7 @@ s32 scx_cid_init(struct scx_sched *sch) u32 cores_per_shard, nr_large; u32 shard_local = 0, cores_in_shard = 0, cids_in_shard = 0; s32 shard_cid, shard_idx; + s32 cluster_cid = -1, cluster_idx = -1; /* llc_scratch = node_scratch & this llc */ cpumask_and(llc_scratch, node_scratch, llc_mask); @@ -268,15 +273,32 @@ s32 scx_cid_init(struct scx_sched *sch) tbls->shard_node[shard_idx] = nid; while (!cpumask_empty(llc_scratch)) { - s32 lcpu = cpumask_first(llc_scratch); - const struct cpumask *sib = topology_sibling_cpumask(lcpu); - s32 core_cid = next_cid; - s32 core_idx = next_core_idx++; - s32 ccpu; + const struct cpumask *sib; + s32 core_cid, core_idx, lcpu, ccpu; u32 max_cores, cids_in_core; - /* core_scratch = llc_scratch & this core */ - cpumask_and(core_scratch, llc_scratch, sib); + /* + * Take the cores of one cluster before moving + * on, so that a cluster is a contiguous cid + * range like the core, LLC and node levels. + */ + if (cpumask_empty(cluster_scratch)) { + s32 xcpu = cpumask_first(llc_scratch); + + cpumask_or(cluster_scratch, topology_cluster_cpumask(xcpu), + topology_sibling_cpumask(xcpu)); + cpumask_and(cluster_scratch, cluster_scratch, llc_scratch); + cluster_cid = next_cid; + cluster_idx = next_cluster_idx++; + } + + lcpu = cpumask_first(cluster_scratch); + sib = topology_sibling_cpumask(lcpu); + core_cid = next_cid; + core_idx = next_core_idx++; + + /* core_scratch = cluster_scratch & this core */ + cpumask_and(core_scratch, cluster_scratch, sib); if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch))) return -EINVAL; @@ -316,8 +338,11 @@ s32 scx_cid_init(struct scx_sched *sch) .node_idx = node_idx, .shard_cid = shard_cid, .shard_idx = shard_idx, + .cluster_cid = cluster_cid, + .cluster_idx = cluster_idx, }; + cpumask_clear_cpu(ccpu, cluster_scratch); cpumask_clear_cpu(ccpu, llc_scratch); cpumask_clear_cpu(ccpu, node_scratch); cpumask_clear_cpu(ccpu, to_walk); @@ -461,8 +486,8 @@ __bpf_kfunc_start_defs(); * starts must be strictly increasing with the first entry 0 and all values < * num_possible_cpus(). The last shard extends to num_possible_cpus() and no * shard may span more than SCX_CID_SHARD_MAX_CPUS cids. Topo info - * (core/LLC/node) is cleared and the shard layout is set from the input. On - * invalid input, abort the scheduler. + * (core/cluster/LLC/node) is cleared and the shard layout is set from the + * input. On invalid input, abort the scheduler. */ __bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid__arena, u32 cpu_to_cid_cnt, const s32 *shard_start__arena, u32 shard_start_cnt, diff --git a/kernel/sched/ext/cid.h b/kernel/sched/ext/cid.h index 2fe2311a0f99..709afbfb97c6 100644 --- a/kernel/sched/ext/cid.h +++ b/kernel/sched/ext/cid.h @@ -13,11 +13,12 @@ * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences * for every op. * - * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or - * NUMA node get contiguous cid ranges, so a topology unit becomes a (start, - * length) slice of cid space. Communication can pass a slice instead of a - * cpumask, and BPF code can process, for example, a u64 word's worth of cids at - * a time. + * cids give every cpu a dense, topology-ordered id. CPUs in each core, + * cluster, LLC or NUMA node get contiguous cid ranges, so a topology unit + * becomes a (start, length) slice of cid space. A core without a wider cluster + * level forms a cluster of its own. Communication can pass a slice + * instead of a cpumask, and BPF code can process, for example, a u64 word's + * worth of cids at a time. * * The mapping is built once at root scheduler enable time by walking the * topology of online cpus only. Going by online cpus is out of necessity: diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h index 139176cf9fc6..38514d6bf149 100644 --- a/kernel/sched/ext/types.h +++ b/kernel/sched/ext/types.h @@ -59,17 +59,31 @@ enum scx_consts { }; /* - * Per-cid topology info. For each topology level (core, LLC, node) and shard, - * records the first cid in the unit and its global index. Global indices are - * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges over - * [0, nr_cores_at_init) with no gaps. No-topo cids have core/LLC/node fields - * set to -1 but always have valid shard assignments. + * Per-cid topology info. For each topology level (core, cluster, LLC, node) and + * shard, records the first cid in the unit and its global index. Global indices + * are consecutive integers assigned in cid-walk order, so e.g. core_idx ranges + * over [0, nr_cores_at_init) with no gaps. No-topo cids have core/cluster/LLC/ + * node fields set to -1 but always have valid shard assignments. + * + * A cluster is the cache-sharing level between the core and the LLC. Where + * this level is absent, each core forms a cluster of its own. Where the cache + * level spans the LLC, the whole LLC forms one cluster. Each cluster has a + * unique cluster_cid and a dense cluster_idx, and its cids form a contiguous + * range. + * + * Every cid therefore has a cluster, so cluster_cid alone does not say whether + * the machine has the level: the first cluster of an LLC starts at the LLC's + * own base cid and a core-wide one at the core's. * * Shards are contiguous CID ranges used as scalable locking/work domains for * sub-scheduler operations. By default each LLC becomes one shard, split into * smaller shards if the LLC exceeds the target size. No-topo cids are packed * into their own max-sized shards. * + * Shards are cut on core boundaries only, so a shard doesn't need to contain a + * whole cluster: a cluster may straddle two shards. Only core, cluster, LLC + * and node are nested, in that order. + * * New fields are appended, never inserted: scx_bpf_cid_topo() copies this * struct out sized by the program's own layout, and an older program's copy * must stay a prefix of the kernel's. @@ -82,6 +96,8 @@ enum scx_consts { * @node_idx: global index of that node, in [0, nr_nodes_at_init) * @shard_cid: first cid of this cid's shard * @shard_idx: global index of that shard, in [0, scx_nr_cid_shards) + * @cluster_cid: first cid of this cid's cluster + * @cluster_idx: global index of that cluster, in [0, nr_clusters_at_init) */ struct scx_cid_topo { s32 core_cid; @@ -92,6 +108,8 @@ struct scx_cid_topo { s32 node_idx; s32 shard_cid; s32 shard_idx; + s32 cluster_cid; + s32 cluster_idx; }; enum scx_cid_consts { From be8be8acb26819cbd3992d8ce007d54899232b31 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:45 +0200 Subject: [PATCH 31/69] sched_ext: Block proxy donors before taking control Proxy execution retains mutex-blocked donors on the runqueue so their scheduling context can execute a lock owner. sched_ext cannot safely inherit such donors unless the BPF scheduler explicitly participates in their admission and ordering. Make sched_ext reject retained donors by default. Implement scx_allow_proxy_exec() to force blocked EXT tasks through the regular block path in schedule(). A donor retained by another scheduling class may already be queued when sched_ext is enabled or when a task switches to SCHED_EXT. Fully block the donor before sched_change_begin() records its queued state. This prevents sched_ext from importing a proxy session which the BPF scheduler did not admit, while leaving ordinary FAIR, RT and DL class transitions untouched. This is a preparatory change to support proxy execution with sched_ext. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 37 +++++++++++++++++++++++++++++++++++-- kernel/sched/ext/ext.h | 2 ++ kernel/sched/syscalls.c | 2 ++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 90cd68c3767f..bddea43fa6a1 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -26,7 +26,22 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); bool scx_allow_proxy_exec(const struct task_struct *p) { - return true; + return p->sched_class != &ext_sched_class; +} + +/* + * End retained proxy execution before sched_ext takes ownership of @p. + * Called with @p's pi and rq locks held immediately before + * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock + * is updated only once. + */ +static void scx_prepare_task_sched_change(struct task_struct *p) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + update_rq_clock(task_rq(p)); + sched_proxy_block_task(task_rq(p), p); } /* @@ -4438,7 +4453,7 @@ int scx_check_setscheduler(struct task_struct *p, int policy) { lockdep_assert_rq_held(task_rq(p)); - /* if disallow, reject transitioning into SCX */ + /* If disallow, reject transitioning into SCX. */ if (scx_enabled() && READ_ONCE(p->scx.disallow) && p->policy != policy && policy == SCHED_EXT) return -EACCES; @@ -4446,6 +4461,20 @@ int scx_check_setscheduler(struct task_struct *p, int policy) return 0; } +/* + * Don't carry a donor retained by another class into sched_ext. The caller + * has updated the rq clock and invokes this immediately before + * sched_change_begin() records the task's queued state. + */ +void scx_prepare_setscheduler(struct task_struct *p, int policy) +{ + lockdep_assert_held(&p->pi_lock); + lockdep_assert_rq_held(task_rq(p)); + + if (scx_enabled() && p->policy != policy && policy == SCHED_EXT) + sched_proxy_block_task(task_rq(p), p); +} + static void process_ddsp_deferred_locals(struct rq *rq) { struct task_struct *p; @@ -7994,6 +8023,10 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (old_class != new_class) queue_flags |= DEQUEUE_CLASS; + if (new_class == &ext_sched_class) { + scx_prepare_task_sched_change(p); + queue_flags |= DEQUEUE_NOCLOCK; + } scoped_guard (sched_change, p, queue_flags) { scx_set_task_slice(p, READ_ONCE(sch->slice_dfl)); diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index cca3f7c97b78..8348d6008651 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -18,6 +18,7 @@ bool scx_can_stop_tick(struct rq *rq); void scx_rq_activate(struct rq *rq); void scx_rq_deactivate(struct rq *rq); int scx_check_setscheduler(struct task_struct *p, int policy); +void scx_prepare_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); bool scx_allow_proxy_exec(const struct task_struct *p); @@ -62,6 +63,7 @@ static inline bool scx_can_stop_tick(struct rq *rq) { return true; } static inline void scx_rq_activate(struct rq *rq) {} static inline void scx_rq_deactivate(struct rq *rq) {} static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; } +static inline void scx_prepare_setscheduler(struct task_struct *p, int policy) {} static inline bool task_on_scx(const struct task_struct *p) { return false; } static inline bool scx_allow_ttwu_queue(const struct task_struct *p) { return true; } static inline bool scx_allow_proxy_exec(const struct task_struct *p) { return true; } diff --git a/kernel/sched/syscalls.c b/kernel/sched/syscalls.c index b215b0ead9a6..45a127deeadd 100644 --- a/kernel/sched/syscalls.c +++ b/kernel/sched/syscalls.c @@ -678,6 +678,8 @@ int __sched_setscheduler(struct task_struct *p, if (prev_class != next_class) queue_flags |= DEQUEUE_CLASS; + scx_prepare_setscheduler(p, policy); + scoped_guard (sched_change, p, queue_flags) { if (!(attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)) { From 5f882051efabaf19f9b4dbe4b50cee1d250ac19e Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:46 +0200 Subject: [PATCH 32/69] sched_ext: Fix ops.running/stopping() pairing for proxy-exec donors With proxy execution, pick_next_task() can select a blocked task as the scheduling context before find_proxy_task() resolves the execution context. From the BPF scheduler perspective, that donor is running while its scheduling context drives the lock owner; ops.tick() and other accounting must therefore remain enclosed by a matching ops.running()/ops.stopping() session. In this scenario, the session boundaries do not always match physical task switches. Keep the "running" session open when the same donor continues on the same CPU. When proxy execution migrates a donor's scheduling context to another CPU, end its running session on the source CPU and start a new session on the destination CPU only after proxy resolution succeeds. This prevents a failed resolution from exposing a provisional ops.running() event. Track these sessions with a new SCX_TASK_RUN_TRACKED flag. The explicit running-state tracking is also required by later donor-based accounting: it prevents an EXT owner executing for a non-EXT donor from being treated as the active EXT scheduling context when it is dequeued. This is a preparatory change for enabling proxy execution together with sched_ext. Acked-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 72 ++++++++++++++----- kernel/sched/ext/internal.h | 6 ++ .../sched_ext/include/scx/enum_defs.autogen.h | 1 + 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index b85676871624..77bc767fdf30 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -104,6 +104,7 @@ enum scx_ent_flags { SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */ SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */ SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ + SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */ /* * Bits 8 to 10 are used to carry task state: diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index bddea43fa6a1..63cbd3977876 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2377,10 +2377,10 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ ops_dequeue(rq, p, deq_flags); /* - * A currently running task which is going off @rq first gets dequeued - * and then stops running. As we want running <-> stopping transitions - * to be contained within runnable <-> quiescent transitions, trigger - * ->stopping() early here instead of in put_prev_task_scx(). + * A current scheduling context which is going off @rq first gets + * dequeued and then stops running. As we want running <-> stopping + * transitions to be contained within runnable <-> quiescent transitions, + * trigger ->stopping() early here instead of in put_prev_task_scx(). * * @p may go through multiple stopping <-> running transitions between * here and put_prev_task_scx() if task attribute changes occur while @@ -2388,11 +2388,13 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ * information meaningful to the BPF scheduler and can be suppressed by * skipping the callbacks if the task is !QUEUED. */ - if (task_current(rq, p) && - (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) { - update_curr_scx(rq); - if (SCX_HAS_OP(sch, stopping)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + if (task_current_donor(rq, p) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) { + if (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq))) { + update_curr_scx(rq); + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, false); + } + p->scx.flags &= ~SCX_TASK_RUN_TRACKED; } if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p)) @@ -3111,10 +3113,23 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev return verdict; } -static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) +static void scx_start_task_running(struct rq *rq, struct task_struct *p) { struct scx_sched *sch = scx_task_sched(p); + if (p->scx.flags & SCX_TASK_RUN_TRACKED) + return; + + if (SCX_HAS_OP(sch, running)) + SCX_CALL_OP_TASK(sch, running, rq, p); + + p->scx.flags |= SCX_TASK_RUN_TRACKED; +} + +static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) +{ + bool first = type == SNT_PICK; + if (type == SNT_REPICK) return; @@ -3129,9 +3144,20 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t p->se.exec_start = rq_clock_task(rq); - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, running) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, running, rq, p); + /* + * See dequeue_task_scx() for why we skip when !QUEUED. + * + * During a normal switch (@first), a blocked task is only a provisional + * donor. Proxy resolution may fail or migrate the donor to another CPU, + * so defer ops.running() until scx_proxy_donor_start() confirms that + * resolution succeeded. + * + * !@first denotes restoration after a SAVE/RESTORE cycle. The matching + * dequeue already issued ops.stopping(), so restart the session here + * regardless of the donor state. + */ + if ((p->scx.flags & SCX_TASK_QUEUED) && !(p->is_blocked && first)) + scx_start_task_running(rq, p); clr_task_runnable(p, true); @@ -3177,6 +3203,12 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t void scx_proxy_donor_start(struct rq *rq) { + struct task_struct *donor = rq->donor; + + lockdep_assert_rq_held(rq); + + if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED)) + scx_start_task_running(rq, donor); } static enum scx_cpu_preempt_reason @@ -3252,9 +3284,17 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, scx_task_slice_ended(rq, p); } - /* see dequeue_task_scx() on why we skip when !QUEUED */ - if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + /* + * Preserve the running session when proxy execution refreshes the same + * donor around an execution-context switch on this rq. + */ + if (next != p && (p->scx.flags & SCX_TASK_QUEUED) && + (p->scx.flags & SCX_TASK_RUN_TRACKED)) { + if (SCX_HAS_OP(sch, stopping)) + SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + + p->scx.flags &= ~SCX_TASK_RUN_TRACKED; + } if (p->scx.flags & SCX_TASK_QUEUED) { set_task_runnable(rq, p); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 0bcceab612de..8439cfa36a78 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -493,6 +493,12 @@ struct sched_ext_ops { * Therefore, always use scx_bpf_task_cpu(@p) to determine the * target CPU the task is going to use. * + * Under proxy execution, the BPF scheduler continues to observe the + * donor as the current scheduling context. A blocked donor enters a + * ->running()/->stopping() session while its scheduling context drives + * the lock owner. The lock owner executing on its behalf is intentionally + * not reported through these callbacks. + * * See ->runnable() for explanation on the task state notifiers. */ void (*running)(struct task_struct *p); diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 92501498f6a3..a6c1e36f8396 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -106,6 +106,7 @@ #define HAVE_SCX_TASK_SUB_INIT #define HAVE_SCX_TASK_IMMED #define HAVE_SCX_TASK_PROTECTED +#define HAVE_SCX_TASK_RUN_TRACKED #define HAVE_SCX_TASK_STATE_SHIFT #define HAVE_SCX_TASK_STATE_BITS #define HAVE_SCX_TASK_STATE_MASK From cea8b60e4aee0435c23d343d31972664c65aa7c3 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:47 +0200 Subject: [PATCH 33/69] sched_ext: Move reject DSQ draining into core The reject DSQ drain will be generalized for placement failures outside sub-scheduler capability rejection. Move the drain into the core sched_ext implementation first so the subsequent behavioral changes can be reviewed separately. No functional change. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 50 ++++++++++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.c | 44 ------------------------------------- kernel/sched/ext/sub.h | 10 --------- 3 files changed, 50 insertions(+), 54 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 63cbd3977876..a4b685d554c2 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4849,6 +4849,56 @@ static void process_deferred_reenq_users(struct rq *rq) } } +#ifdef CONFIG_EXT_SUB_SCHED +/* + * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides + * from p->scx.reenq_reason_*. + * + * A task can be re-rejected repeatedly. The reenqueue is bounded per task in + * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. + * Rejection can't happen for root. + */ +static void scx_reenq_reject(struct rq *rq) +{ + LIST_HEAD(tasks); + struct task_struct *p, *n; + + lockdep_assert_rq_held(rq); + + if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + return; + + /* + * Move to a private list so a task re-rejected by the + * scx_do_enqueue_task() below isn't revisited this round. + */ + list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { + /* migration_pending tasks should have bypassed to local DSQ */ + if (WARN_ON_ONCE(p->migration_pending)) + continue; + + scx_reenq_wait_dispatching(p); + scx_dispatch_dequeue(rq, p); + + if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + p->scx.flags |= SCX_TASK_REENQ_CAP; + + list_add_tail(&p->scx.dsq_list.node, &tasks); + } + + list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { + list_del_init(&p->scx.dsq_list.node); + + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); + + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + } +} +#else +static void scx_reenq_reject(struct rq *rq) {} +#endif + static void run_deferred(struct rq *rq) { process_ddsp_deferred_locals(rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 10567196be96..abe10a3529b9 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -772,50 +772,6 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) return true; } -/* - * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides - * from p->scx.reenq_reason_*. - * - * A task can be re-rejected repeatedly. The reenqueue is bounded per task in - * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. - * Rejection can't happen for root. - */ -void __scx_reenq_reject(struct rq *rq) -{ - LIST_HEAD(tasks); - struct task_struct *p, *n; - - if (list_empty(&rq->scx.reject_dsq.list)) - return; - - /* - * Move to a private list so a task re-rejected by the - * scx_do_enqueue_task() below isn't revisited this round. - */ - list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { - /* migration_pending tasks should have bypassed to local DSQ */ - if (WARN_ON_ONCE(p->migration_pending)) - continue; - - scx_reenq_wait_dispatching(p); - scx_dispatch_dequeue(rq, p); - - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= SCX_TASK_REENQ_CAP; - - list_add_tail(&p->scx.dsq_list.node, &tasks); - } - - list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) { - list_del_init(&p->scx.dsq_list.node); - - scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - } -} - /* record a caps change, see struct scx_caps_updated */ static void caps_updated_record(struct scx_pshard *ps, const struct scx_cmask *cids, u64 caps, struct list_head *to_deliver) diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 6a7076b9a1dd..7621c28c7e8c 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -37,7 +37,6 @@ void scx_discard_stale_ecaps_syncs(void); struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags); bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p); -void __scx_reenq_reject(struct rq *rq); void scx_rescue_charge(struct rq *rq, s64 delta_exec); void scx_rescue_end(struct rq *rq); bool scx_rescue_keep(struct rq *rq, struct task_struct *p); @@ -91,14 +90,6 @@ static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch return __scx_resolve_local_dsq(sch, rq, p, enq_flags); } -static inline void scx_reenq_reject(struct rq *rq) -{ - lockdep_assert_rq_held(rq); - - if (scx_has_subs()) - __scx_reenq_reject(rq); -} - #else /* CONFIG_EXT_SUB_SCHED */ static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } @@ -120,7 +111,6 @@ static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcp static inline void scx_discard_stale_ecaps_syncs(void) {} static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; } static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; } -static inline void scx_reenq_reject(struct rq *rq) {} static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {} static inline void scx_rescue_end(struct rq *rq) {} static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } From 4eeabe0a3ef7069ba00d450b8b21fd5e0fe76e13 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:48 +0200 Subject: [PATCH 34/69] sched_ext: Generalize the reject DSQ reenqueue path The reject DSQ is currently specific to sub-scheduler cap failures. Its storage and initialization depend on CONFIG_EXT_SUB_SCHED, and the drain path assumes every rejected task was rejected for SCX_TASK_REENQ_CAP. Other transient placement failures need the same ability to park a task on its source rq and return it to the owning BPF scheduler. Make the reject DSQ unconditional and carry the reenqueue reason directly in p->scx.flags from the rejection site. A reenqueue reason remains valid while ops.enqueue() runs so that BPF can inspect it. Clear it immediately after the callback returns and before resolving a direct dispatch. Paths which bypass ops.enqueue() clear the reason before the kernel-selected placement. A rejection from the new placement can therefore install its reason into a clear field. Route all paths through a common cleanup point to enforce this ordering. This is a preparatory change to support proxy execution with sched_ext. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 91 +++++++++++++++++++++++------------------- kernel/sched/ext/sub.c | 2 + kernel/sched/sched.h | 2 +- 3 files changed, 53 insertions(+), 42 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index a4b685d554c2..9a4a5afee211 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1672,12 +1672,10 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq, struct scx_dispatch_q *dsq, struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags) { - bool is_rq_owned = false; + bool is_rq_owned = dsq_is_rq_owned(dsq); - if (dsq->id == SCX_DSQ_LOCAL) { + if (dsq->id == SCX_DSQ_LOCAL) dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags); - is_rq_owned = true; - } WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node)); WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) || @@ -2065,6 +2063,12 @@ bool scx_rq_online(struct rq *rq) void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu) { + enum { + ENQ_ACTION_NONE, + ENQ_ACTION_DIRECT, + ENQ_ACTION_LOCAL_NOREFILL, + ENQ_ACTION_ENQUEUE, + } action = ENQ_ACTION_NONE; struct scx_sched *sch = scx_task_sched(p); struct task_struct **ddsp_taskp; struct scx_dispatch_q *dsq; @@ -2098,7 +2102,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, __scx_exit(sch, SCX_EXIT_ERROR_REENQ, 0, cpu_of(rq), "%s[%d] reenqueued %u times without running", p->comm, p->pid, p->scx.reenq_cnt); - return; + goto out; } } @@ -2163,14 +2167,14 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, * dequeue may be waiting. The store_release matches their load_acquire. */ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq); - return; + goto out; direct: - direct_dispatch(sch, p, enq_flags); - return; + action = ENQ_ACTION_DIRECT; + goto out; local_norefill: - scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags); - return; + action = ENQ_ACTION_LOCAL_NOREFILL; + goto out; local: dsq = &rq->scx.local_dsq; goto enqueue; @@ -2182,9 +2186,27 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, goto enqueue; enqueue: - refill_task_slice_dfl(sch, p); - clear_direct_dispatch(p); - scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + action = ENQ_ACTION_ENQUEUE; +out: + /* The reason is input to ops.enqueue(), not to the resulting placement. */ + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + + switch (action) { + case ENQ_ACTION_DIRECT: + direct_dispatch(sch, p, enq_flags); + break; + case ENQ_ACTION_LOCAL_NOREFILL: + scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, + enq_flags); + break; + case ENQ_ACTION_ENQUEUE: + refill_task_slice_dfl(sch, p); + clear_direct_dispatch(p); + scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); + break; + case ENQ_ACTION_NONE: + break; + } } static bool task_runnable(const struct task_struct *p) @@ -2410,6 +2432,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ sub_nr_running(rq, 1); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; /* see scx_task_slice_ended() for the save/restore exception */ if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) @@ -3140,6 +3163,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t */ ops_dequeue(rq, p, SCX_DEQ_CORE_SCHED_EXEC); scx_dispatch_dequeue(rq, p); + p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } p->se.exec_start = rq_clock_task(rq); @@ -3314,7 +3338,6 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (p->scx.flags & SCX_TASK_IMMED) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } else { u64 enq_flags = 0; @@ -4655,8 +4678,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; list_add_tail(&p->scx.dsq_list.node, &tasks); @@ -4667,7 +4689,6 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; nr_enqueued++; } @@ -4780,14 +4801,11 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag dispatch_dequeue_locked(p, dsq); raw_spin_unlock(&dsq->lock); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); p->scx.flags |= reason; scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { scx_rq_lock_drop(locked_rq); raw_spin_rq_unlock(locked_rq); @@ -4849,14 +4867,14 @@ static void process_deferred_reenq_users(struct rq *rq) } } -#ifdef CONFIG_EXT_SUB_SCHED /* - * Drain @rq->scx.reject_dsq, reenqueueing each task so the BPF re-decides - * from p->scx.reenq_reason_*. + * Drain @rq->scx.reject_dsq and reenqueue each task so that its owning BPF + * scheduler chooses placement again. * - * A task can be re-rejected repeatedly. The reenqueue is bounded per task in - * scx_do_enqueue_task(), which ejects the owning sub past SCX_REENQ_MAX_REPEAT. - * Rejection can't happen for root. + * A task can be re-rejected repeatedly. Reenqueues are bounded per task by + * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning + * scheduler. The private list below prevents a task from being revisited in + * the same round. */ static void scx_reenq_reject(struct rq *rq) { @@ -4865,25 +4883,21 @@ static void scx_reenq_reject(struct rq *rq) lockdep_assert_rq_held(rq); - if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list)) + if (list_empty(&rq->scx.reject_dsq.list)) return; /* - * Move to a private list so a task re-rejected by the + * Move tasks to a private list so a task re-rejected by * scx_do_enqueue_task() below isn't revisited this round. */ list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { /* migration_pending tasks should have bypassed to local DSQ */ - if (WARN_ON_ONCE(p->migration_pending)) - continue; + WARN_ON_ONCE(p->migration_pending); + WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)); scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); - if (WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)) - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; - p->scx.flags |= SCX_TASK_REENQ_CAP; - list_add_tail(&p->scx.dsq_list.node, &tasks); } @@ -4891,13 +4905,8 @@ static void scx_reenq_reject(struct rq *rq) list_del_init(&p->scx.dsq_list.node); scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); - - p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; } } -#else -static void scx_reenq_reject(struct rq *rq) {} -#endif static void run_deferred(struct rq *rq) { @@ -9015,8 +9024,8 @@ void __init init_sched_ext_class(void) /* local_dsq's sch will be set during scx_root_enable() */ BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL)); -#ifdef CONFIG_EXT_SUB_SCHED BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL)); +#ifdef CONFIG_EXT_SUB_SCHED scx_rescue_init(rq); #endif diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index abe10a3529b9..b5989aaffcc6 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -750,6 +750,8 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq p->scx.reenq_reason_caps = missing; p->scx.reenq_reason_cid = cid; + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + p->scx.flags |= SCX_TASK_REENQ_CAP; return &rq->scx.reject_dsq; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d41d23a5632b..1df76a3f38b9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -810,8 +810,8 @@ struct scx_rq_rescue { struct scx_rq { struct scx_dispatch_q local_dsq; + struct scx_dispatch_q reject_dsq; /* staging for rejected tasks */ #ifdef CONFIG_EXT_SUB_SCHED - struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */ struct scx_rq_rescue rescue; #endif struct list_head runnable_list; /* runnable tasks on this rq */ From 13ae184e4b8279dcc7b84bf58e987a60d9aad69a Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:49 +0200 Subject: [PATCH 35/69] sched_ext: Handle proxy-exec races in remote DSQ transfers Without proxy execution, the DSQ lock and holding_cpu handshake ensure that a task cannot be dequeued or start running during an rq-lock handoff without clearing holding_cpu. Proxy execution is an exception: a task can start physically executing as a lock owner while its scheduling context remains on a DSQ; its on-CPU or migration-disabled state can therefore change without clearing holding_cpu. Recheck these states after acquiring the source rq lock. If the transfer can no longer proceed, park the task on the source rq's reject DSQ and reenqueue it through its owning scheduler. This preserves the BPF scheduler's placement policy and keeps descendant tasks within their sub-scheduler's cap grants. Queue a deferred drain for every rejection. If a drain finds a proxy-rejected task still running or donating, leave it parked and record a retry request on the rq. scx_proxy_reenqueue_retry() consumes the request and schedules another drain when proxy state changes. Without this change and proxy execution enabled, stress-ng --pipeherd can trigger this race and migrate an active execution context, leading to sleeping-while-atomic warnings and subsequent lockdep corruption. This is a preparatory change to support proxy execution with sched_ext. Suggested-by: Tejun Heo Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- include/linux/sched/ext.h | 2 + kernel/sched/ext/ext.c | 143 +++++++++++++++--- kernel/sched/ext/internal.h | 9 ++ kernel/sched/ext/sub.c | 4 +- kernel/sched/sched.h | 1 + .../sched_ext/include/scx/enum_defs.autogen.h | 1 + 6 files changed, 140 insertions(+), 20 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 77bc767fdf30..23f9e178bc5a 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -137,6 +137,7 @@ enum scx_ent_flags { * IMMED reenqueued due to failed ENQ_IMMED * PREEMPTED preempted while running * CAP sub-sched cap miss, see p->scx.reenq_reason_* + * PROXY proxy state prevented a remote DSQ transfer */ SCX_TASK_REENQ_REASON_SHIFT = 12, SCX_TASK_REENQ_REASON_BITS = 3, @@ -147,6 +148,7 @@ enum scx_ent_flags { SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_PROXY = 5 << SCX_TASK_REENQ_REASON_SHIFT, /* iteration cursor, not a task */ SCX_TASK_CURSOR = 1 << 31, diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9a4a5afee211..7ae4e6d0e2e4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1138,8 +1138,17 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +/* + * Retry proxy-rejected tasks which couldn't be reenqueued by an earlier drain. + */ void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) { + lockdep_assert_rq_held(rq); + + if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { + rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; + schedule_deferred_locked(rq); + } } void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, @@ -1600,8 +1609,10 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, SCX_CALL_OP_TASK(sch, dequeue, rq, p, 0); /* - * Only local inserts get the wakeup treatment below. Rejects kick the - * deferred reenq and rescue parks are paced by the rescue timer. + * Only local inserts get the wakeup treatment below. Rejects kick a + * deferred reenq and rescue parks are paced by the rescue timer. Proxy + * rejects which aren't ready when drained request a later retry from + * scx_proxy_reenqueue_retry(). */ if (unlikely(dsq->id != SCX_DSQ_LOCAL)) { if (dsq->id == SCX_DSQ_REJECT) @@ -2573,8 +2584,10 @@ static void move_remote_task_to_local_dsq(struct scx_sched *sch, * - The BPF scheduler is bypassed while the rq is offline and we can always say * no to the BPF scheduler initiated migrations while offline. * - * The caller must ensure that @p and @rq are on different CPUs. - * If enforce == true, caller must hold @p's rq lock. + * The caller must ensure that @p and @rq are on different CPUs. If @enforce is + * true, report violations attributable to BPF-directed migrations. The caller + * must hold @p's rq lock to avoid reporting a transient race as a scheduler + * error. */ static bool task_can_run_on_remote_rq(struct scx_sched *sch, struct task_struct *p, struct rq *rq, @@ -2582,11 +2595,6 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, { s32 cpu = cpu_of(rq); - /* - * To prevent races with @p still running on its old CPU while switching - * out, make sure we're holding @p's rq lock so as not to risk - * erroneously killing the BPF scheduler. - */ if (enforce) lockdep_assert_rq_held(task_rq(p)); @@ -2633,6 +2641,64 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, return true; } +/* + * Proxy execution can change @p's execution and migration-disabled state + * without touching its DSQ entry or clearing holding_cpu. Check those states + * with @p's rq locked. Without proxy execution, the holding_cpu handshake is + * sufficient and this must not affect the existing migration path. + * + * A BPF-directed transfer to a remote local DSQ performs a normal task + * migration and thus cannot move a migration-disabled task. In contrast, + * proxy_migrate_task() moves only a blocked donor's scheduling context towards + * the mutex owner and preserves its execution home in wake_cpu. The latter is + * therefore allowed even when the donor is migration-disabled. + */ +static bool task_proxy_running_or_donating(struct task_struct *p) +{ + struct rq *src_rq = task_rq(p); + + lockdep_assert_rq_held(src_rq); + + if (!sched_proxy_exec()) + return false; + + /* @p may be rq->curr under another task's scheduling context. */ + if (task_on_cpu(src_rq, p)) + return true; + + /* Don't move an active scheduling context off its source rq. */ + if (task_current_donor(src_rq, p)) + return true; + + return false; +} + +static bool task_proxy_unsafe_to_move(struct task_struct *p) +{ + if (!sched_proxy_exec()) + return false; + + return task_proxy_running_or_donating(p) || is_migration_disabled(p); +} + +/* + * Park a task whose remote transfer raced with proxy execution. Reenqueueing + * from the source rq makes the task's owning scheduler choose its placement + * again and preserves sub-scheduler containment. + */ +static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 enq_flags) +{ + lockdep_assert_rq_held(rq); + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + + p->scx.holding_cpu = -1; + p->scx.flags |= SCX_TASK_REENQ_PROXY; + scx_divert_strip_flags(p, &enq_flags); + + scx_dispatch_enqueue(sch, rq, &rq->scx.reject_dsq, p, 0, 0, enq_flags); +} + /** * unlink_dsq_and_switch_rq_lock() - Unlink task and switch to its rq lock * @p: target task @@ -2700,6 +2766,19 @@ static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, struct scx_dispatch_q *dsq, struct rq *src_rq) { if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { + /* + * Proxy execution may have changed @p's running or + * migration-disabled state while switching rq locks without + * clearing holding_cpu. Park it on the source rq and let its + * owning scheduler choose its placement again. + */ + if (unlikely(task_proxy_unsafe_to_move(p))) { + p->scx.dsq = NULL; + scx_proxy_reject_task(sch, src_rq, p, enq_flags | SCX_ENQ_CLEAR_OPSS); + switch_rq_lock(src_rq, this_rq); + return false; + } + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); return true; } else { @@ -2737,6 +2816,19 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, if (dst_dsq->id == SCX_DSQ_LOCAL) { dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq); + /* + * Unlike the rq-lock handoff paths, @src_rq has been locked + * throughout this operation. Only active proxy state can race the + * move here; let the enforcing check below diagnose an ordinary + * migration-disabled task. + */ + if (src_rq != dst_rq && + unlikely(task_proxy_running_or_donating(p))) { + dispatch_dequeue_locked(p, src_dsq); + raw_spin_unlock(&src_dsq->lock); + scx_proxy_reject_task(sch, src_rq, p, enq_flags); + return src_rq; + } if (src_rq != dst_rq && unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { dst_dsq = find_global_dsq(sch, task_cpu(p)); @@ -2894,6 +2986,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, if (likely(p->scx.holding_cpu == raw_smp_processor_id()) && !WARN_ON_ONCE(src_rq != task_rq(p))) { bool fallback = false; + /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by @@ -2903,6 +2996,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, p->scx.holding_cpu = -1; scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); + } else if (unlikely(task_proxy_unsafe_to_move(p))) { + fallback = true; + scx_proxy_reject_task(sch, src_rq, p, enq_flags); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { p->scx.holding_cpu = -1; fallback = true; @@ -4868,13 +4964,13 @@ static void process_deferred_reenq_users(struct rq *rq) } /* - * Drain @rq->scx.reject_dsq and reenqueue each task so that its owning BPF - * scheduler chooses placement again. + * Drain ready tasks from @rq->scx.reject_dsq and reenqueue them so that their + * owning BPF schedulers choose placement again. Proxy-active tasks remain + * parked and rearm the retry notification for a later proxy resolution. * * A task can be re-rejected repeatedly. Reenqueues are bounded per task by * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning - * scheduler. The private list below prevents a task from being revisited in - * the same round. + * scheduler. */ static void scx_reenq_reject(struct rq *rq) { @@ -4887,13 +4983,26 @@ static void scx_reenq_reject(struct rq *rq) return; /* - * Move tasks to a private list so a task re-rejected by + * Move ready tasks to a private list so a task re-rejected by * scx_do_enqueue_task() below isn't revisited this round. */ list_for_each_entry_safe(p, n, &rq->scx.reject_dsq.list, scx.dsq_list.node) { - /* migration_pending tasks should have bypassed to local DSQ */ - WARN_ON_ONCE(p->migration_pending); - WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_REENQ_REASON_MASK)); + u32 reason = p->scx.flags & SCX_TASK_REENQ_REASON_MASK; + + WARN_ON_ONCE(!reason); + + if (sched_proxy_exec() && reason == SCX_TASK_REENQ_PROXY) { + /* Affinity machinery will dequeue and reactivate @p. */ + if (p->migration_pending) + continue; + + if (task_on_cpu(rq, p) || task_current_donor(rq, p)) { + rq->scx.flags |= SCX_RQ_PROXY_RETRY; + continue; + } + } else { + WARN_ON_ONCE(p->migration_pending); + } scx_reenq_wait_dispatching(p); scx_dispatch_dequeue(rq, p); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 8439cfa36a78..deb1a2b8ab9b 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1824,6 +1824,15 @@ enum scx_enq_flags { SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */ }; +/* Strip priority and carried slice state when diverting from a local DSQ. */ +static inline void scx_divert_strip_flags(struct task_struct *p, u64 *enq_flags) +{ + *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | + SCX_ENQ_PREEMPT_LAZY | SCX_ENQ_HEAD | + SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); + p->scx.flags &= ~SCX_TASK_IMMED; +} + enum scx_deq_flags { /* expose select DEQUEUE_* flags as enums */ SCX_DEQ_SLEEP = DEQUEUE_SLEEP, diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index b5989aaffcc6..08599bc57801 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -733,9 +733,7 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq * or HEAD - a diversion has no priority and IMMED is not allowed on * non-local DSQs. Strip the enq and task flags along with the slice. */ - *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY | - SCX_ENQ_HEAD | SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL); - p->scx.flags &= ~SCX_TASK_IMMED; + scx_divert_strip_flags(p, enq_flags); /* the enqueuer opted for rescue instead of rejection and reenqueue */ if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1df76a3f38b9..25e20f82788a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -791,6 +791,7 @@ enum scx_rq_flags { SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ + SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_DISPATCH = 1 << 17, diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index a6c1e36f8396..015eb6f40068 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -124,6 +124,7 @@ #define HAVE_SCX_TASK_REENQ_IMMED #define HAVE_SCX_TASK_REENQ_PREEMPTED #define HAVE_SCX_TASK_REENQ_CAP +#define HAVE_SCX_TASK_REENQ_PROXY #define HAVE_SCX_TASK_CURSOR #define HAVE_SCX_ECODE_RSN_HOTPLUG #define HAVE_SCX_ECODE_RSN_CGROUP_OFFLINE From 3d6577ea93c9633d95430c82c1cfe189d0f8fd47 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:50 +0200 Subject: [PATCH 36/69] sched_ext: Split curr|donor references properly With proxy execution, the task selected by the scheduler and the task physically executing can differ. A blocked mutex waiter donates its scheduling context to the lock owner: D -----------------> M -------------> O ----------------> T [donor] blocked on [mutex] owned by [owner] preempted by [task] \_________________________________^ donates scheduling context where: D = blocked donor M = mutex O = mutex owner T = competing runnable task During a proxy execution switch, D supplies the scheduling class, priority, and runtime budget, while O supplies the execution context: O is the task whose code physically executes. T is a competing runnable task which may preempt the D/O proxy execution. Consider FAIR and EXT tasks with sched_ext running in partial mode. FAIR can be replaced with a higher scheduling class such as RT or deadline without changing the class interaction described here. An active proxy pair can only have a donor in the same scheduling class as the owner or in a higher-priority class. If the owner belongs to a higher-priority class, pick_next_task() selects the owner directly before considering the donor, so find_proxy_task() is not entered for that donor. The blocked-on relationship may still exist, but it does not result in proxy execution. The relevant combinations are: 1. D is EXT, O is EXT, T is EXT D can interrupt T according to BPF scheduling policy. O executes with D's EXT priority and runtime budget, while T waits in EXT. 2. D is EXT, O is EXT, T is FAIR D is visible to the BPF scheduler, but cannot preempt T because EXT is below FAIR. Once T stops, BPF can dispatch D and O executes with D's EXT priority and runtime budget. If T becomes runnable again, it preempts the D/O proxy execution. 3. D is FAIR, O is EXT, T is EXT D preempts T as the higher-class scheduling context. O executes with D's FAIR priority and runtime budget, while T waits in EXT. D is not visible to the BPF scheduler. 4. D is FAIR, O is EXT, T is FAIR D competes with T according to its FAIR deadline. When D is selected, O executes with D's FAIR priority and runtime budget. D is not visible to the BPF scheduler. 5. D is FAIR, O is FAIR, T is EXT The D/O proxy execution has a FAIR scheduling context, so T cannot preempt it from the lower EXT class. 6. D is FAIR, O is FAIR, T is FAIR O, T, and D all have FAIR scheduling contexts. D remains runnable as a blocked proxy donor. When CFS selects D, O executes using D's FAIR scheduling context. When CFS selects O, O executes using its own FAIR context, and when CFS selects T, T executes normally. D is not visible to the BPF scheduler. Thus, sched_ext policy and accounting must generally use rq->donor, the scheduler-selected task which supplies the scheduling context, rather than rq->curr, the task whose code physically executes. Without proxy execution they are the same task. On nohz_full CPUs, keep the tick running whenever the selected donor is blocked. A selected blocked donor is necessarily awaiting proxy execution, so this avoids relying on the transient relationship between rq->curr and rq->donor during context switches. It also leaves ordinary sched_ext context switches and non-nohz_full CPUs untouched. Allowing proxy execution to run tickless remains a future improvement. Moreover, extend scx_dump_state() to report both contexts. Each CPU record now includes a donor= line. If an EXT donor differs from rq->curr, also emit its detailed task record. The existing '*' marker continues to identify rq->curr, while the donor= line identifies the otherwise unmarked donor record. Note that at this point in the series, CONFIG_SCHED_PROXY_EXEC still depends on !CONFIG_SCHED_CLASS_EXT, so proxy execution and sched_ext cannot be enabled together. The scheduling changes are therefore preparatory. A later patch removes this restriction. Co-developed-by: John Stultz Signed-off-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- Documentation/scheduler/sched-ext.rst | 13 +++ kernel/sched/ext/ext.c | 132 ++++++++++++++++---------- kernel/sched/ext/sub.c | 5 +- kernel/sched/ext/sub.h | 11 ++- 4 files changed, 106 insertions(+), 55 deletions(-) diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index 794ae80b3ba3..13ba2cb7831c 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -503,6 +503,19 @@ and edge cases, to name a few examples: class, in which case it will exit the tick-dispatch loop even though it is runnable and has a non-zero slice. +* Under proxy execution, sched_ext continues to observe the donor as the + current scheduling context. Accordingly, ``ops.running()`` and + ``ops.stopping()`` report when the donor's scheduling context becomes active + and inactive, even when the donor is blocked and a lock owner executes on its + behalf. The physical execution context is intentionally not reported through + these callbacks. + + A blocked donor enters a running session only after proxy resolution finds + an execution context. The session remains active if only the physical + execution context changes while the donor remains the same. Running sessions + are tracked so that ``ops.running()`` and ``ops.stopping()`` remain paired + and are not emitted recursively. + See the "Scheduling Cycle" section for a more detailed description of how a freshly woken up task gets on a CPU. diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 7ae4e6d0e2e4..9d41ec82f3bf 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -425,28 +425,28 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) */ /* - * If we're in the dispatch path holding rq lock, $curr may or may not + * If we're in the dispatch path holding rq lock, $donor may or may not * be ready depending on whether the on-going dispatch decides to extend - * $curr's slice. We say yes here and resolve it at the end of dispatch. + * $donor's slice. We say yes here and resolve it at the end of dispatch. * See dispatch_one(). */ if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return true; /* - * The preemption flags clear $curr's slice if on SCX and kick dispatch, - * so allow them to avoid spuriously triggering reenq on a combined + * The preemption flags clear $donor's slice if on SCX and kick dispatch, + * so allow it to avoid spuriously triggering reenq on a combined * PREEMPT|IMMED insertion. */ if (enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) { - struct task_struct *curr = rq->curr; + struct task_struct *donor = rq->donor; /* * A protected slice refuses the preemption and the cpu stays * occupied. See rq_owned_post_enq(). */ - return curr->sched_class != &ext_sched_class || - likely(!(curr->scx.flags & SCX_TASK_PROTECTED)); + return donor->sched_class != &ext_sched_class || + likely(!(donor->scx.flags & SCX_TASK_PROTECTED)); } /* @@ -1462,20 +1462,27 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e static void update_curr_scx(struct rq *rq) { - struct task_struct *curr = rq->curr; + struct task_struct *donor; s64 delta_exec; + /* + * update_curr_scx() is selected through rq->donor->sched_class, not + * rq->curr->sched_class, so @donor is always an EXT task here. If an EXT + * owner executes for a FAIR donor, FAIR's update_curr() runs instead. + */ + donor = rq->donor; + /* apply even on 0 delta_exec, callers may still act on the slice */ - apply_task_slice_oob(rq, curr); + apply_task_slice_oob(rq, donor); delta_exec = update_curr_common(rq); if (unlikely(delta_exec <= 0)) return; - if (curr->scx.slice != SCX_SLICE_INF) - curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); + if (donor->scx.slice != SCX_SLICE_INF) + donor->scx.slice -= min_t(u64, donor->scx.slice, delta_exec); - if (unlikely(curr == scx_rescuee(rq))) + if (unlikely(donor == scx_rescuee(rq))) scx_rescue_charge(rq, delta_exec); dl_server_update(&rq->ext_server, delta_exec); @@ -1666,9 +1673,9 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq, if (rq->scx.flags & SCX_RQ_IN_DISPATCH) return; - if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && p != rq->curr && - rq->curr->sched_class == &ext_sched_class) { - if (likely(scx_set_task_slice(rq->curr, 0))) { + if ((enq_flags & (SCX_ENQ_PREEMPT | SCX_ENQ_PREEMPT_LAZY)) && + p != rq->donor && rq->donor->sched_class == &ext_sched_class) { + if (likely(scx_set_task_slice(rq->donor, 0))) { if (enq_flags & SCX_ENQ_PREEMPT) resched_curr(rq); else @@ -2273,13 +2280,14 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ rq->scx.flags |= SCX_RQ_IN_WAKEUP; /* - * Restoring a running task will be immediately followed by - * set_next_task_scx() which expects the task to not be on the BPF + * Restoring the current scheduling context will be immediately followed + * by set_next_task_scx() which expects the task to not be on the BPF * scheduler as tasks can only start running through local DSQs. Force * direct-dispatch into the local DSQ by setting the sticky_cpu. Mark * IGNORE_CAPS to force entry into the local DSQ. */ - if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p)) { + if (unlikely(enq_flags & ENQUEUE_RESTORE) && + task_current_donor(rq, p)) { sticky_cpu = cpu_of(rq); enq_flags |= SCX_ENQ_IGNORE_CAPS; } @@ -2446,7 +2454,7 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_ p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK; /* see scx_task_slice_ended() for the save/restore exception */ - if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p))) + if (!((deq_flags & DEQUEUE_SAVE) && task_current_donor(rq, p))) scx_task_slice_ended(rq, p); clear_direct_dispatch(p); @@ -3014,7 +3022,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, } /* if the destination CPU is idle, wake it up */ - if (!fallback && sched_class_above(p->sched_class, dst_rq->curr->sched_class)) + if (!fallback && sched_class_above(p->sched_class, + dst_rq->donor->sched_class)) resched_curr(dst_rq); } @@ -3248,6 +3257,7 @@ static void scx_start_task_running(struct rq *rq, struct task_struct *p) static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) { bool first = type == SNT_PICK; + bool can_stop_tick; if (type == SNT_REPICK) return; @@ -3283,6 +3293,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t /* apply any pending out-of-band slice request before the tick decision */ apply_task_slice_oob(rq, p); + can_stop_tick = p->scx.slice == SCX_SLICE_INF && !p->is_blocked; /* * @p is getting newly scheduled or got kicked after someone updated its @@ -3293,7 +3304,7 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t * nohz. In the future, we might want to add a mechanism to update * load_avgs periodically on tick-stopped CPUs. */ - if (p->scx.slice == SCX_SLICE_INF) { + if (can_stop_tick) { if (!(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) { /* * Bypass mode always assigns finite slices, so @p @@ -3314,7 +3325,8 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t /* * @rq still references the outgoing scheduling context. A finite - * slice is sufficient by itself to require the tick. + * slice or a blocked proxy donor is sufficient by itself to require + * the tick. */ if (tick_nohz_full_cpu(cpu_of(rq))) tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); @@ -3633,7 +3645,7 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r static struct task_struct * do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) { - struct task_struct *prev = rq->curr; + struct task_struct *prev = rq->donor; enum scx_dsp_verdict verdict; struct task_struct *p; @@ -4079,9 +4091,9 @@ void scx_tick(struct rq *rq) update_other_load_avgs(rq); } -static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) +static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued) { - struct scx_sched *sch = scx_task_sched(curr); + struct scx_sched *sch = scx_task_sched(donor); update_curr_scx(rq); @@ -4090,13 +4102,14 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) * management. */ if (scx_bypassing(sch, cpu_of(rq))) - scx_set_task_slice(curr, 0); + scx_set_task_slice(donor, 0); else if (SCX_HAS_OP(sch, tick)) - SCX_CALL_OP_TASK(sch, tick, rq, curr); + SCX_CALL_OP_TASK(sch, tick, rq, donor); - if (!curr->scx.slice) { + if (!donor->scx.slice) { /* the slice can't be trusted while bypassing */ - if (READ_ONCE(curr->scx.lazy_resched) && !scx_bypassing(sch, cpu_of(rq))) + if (READ_ONCE(donor->scx.lazy_resched) && + !scx_bypassing(sch, cpu_of(rq))) scx_resched_curr_lazy(rq); else resched_curr(rq); @@ -4789,16 +4802,16 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) } /* - * The revoke that scheduled this scan may have raced the pick: curr + * The revoke that scheduled this scan may have raced the pick: donor * may be a now-capless task, either one that kept running or one * promoted off the local DSQ between the ecaps sync and this scan. * Zero the slice to evict it. The enqueue gate blocks new capless * inserts, so no later pick can slip through after the scan. */ if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && - rq->curr->sched_class == &ext_sched_class && - scx_task_reenq_on_cap_revoke(rq, rq->curr)) { - scx_set_task_slice(rq->curr, 0); + rq->donor->sched_class == &ext_sched_class && + scx_task_reenq_on_cap_revoke(rq, rq->donor)) { + scx_set_task_slice(rq->donor, 0); resched_curr(rq); } @@ -5033,14 +5046,18 @@ static void run_deferred(struct rq *rq) #ifdef CONFIG_NO_HZ_FULL bool scx_can_stop_tick(struct rq *rq) { - struct task_struct *p = rq->curr; + struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); + /* Keep the tick running while a blocked proxy donor is selected. */ + if (p->is_blocked) + return false; + if (p->sched_class != &ext_sched_class) return true; /* - * @rq->curr may still reference an outgoing EXT task after it has been + * @rq->donor may still reference an outgoing EXT task after it has been * dequeued. If no EXT tasks are accounted on @rq, ignore its stale * slice state. If another task is dispatched from a DSQ, * set_next_task_scx() will update the dependency for the incoming task. @@ -5061,7 +5078,8 @@ bool scx_can_stop_tick(struct rq *rq) /* * @rq can dispatch from different DSQs, so we can't tell whether it * needs the tick or not by looking at nr_running. Allow stopping ticks - * iff the BPF scheduler indicated so. See set_next_task_scx(). + * iff set_next_task_scx() determined that the selected scheduling context + * can run tickless. */ return rq->scx.flags & SCX_RQ_CAN_STOP_TICK; } @@ -6576,9 +6594,9 @@ void scx_bypass(struct scx_sched *sch, bool bypass) /* * Bypass trumps protection. Cycling clears for queued - * tasks but current task needs explicit stripping. + * tasks but the current donor needs explicit stripping. */ - if (bypass && task_current(rq, p)) + if (bypass && task_current_donor(rq, p)) scx_task_slice_ended(rq, p); /* cycling deq/enq is enough, see the function comment */ @@ -7286,6 +7304,8 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, scx_rescue_dump(&ns, rq); scx_dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); + scx_dump_line(&ns, " donor=%s[%d] class=%ps", + rq->donor->comm, rq->donor->pid, rq->donor->sched_class); if (!cpumask_empty(pcpu->cpus_to_kick)) scx_dump_line(&ns, " cpus_to_kick : %*pb", cpumask_pr_args(pcpu->cpus_to_kick)); @@ -7332,6 +7352,10 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, if (rq->curr->sched_class == &ext_sched_class && (dump_all_tasks || scx_task_on_sched(sch, rq->curr))) scx_dump_task(sch, s, dctx, rq, rq->curr, '*'); + if (rq->donor != rq->curr && + rq->donor->sched_class == &ext_sched_class && + (dump_all_tasks || scx_task_on_sched(sch, rq->donor))) + scx_dump_task(sch, s, dctx, rq, rq->donor, ' '); list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) if (dump_all_tasks || scx_task_on_sched(sch, p)) @@ -8890,7 +8914,7 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r bool preempt, preempt_lazy, wait, immediate; rq_lock_irqsave(rq, &rf); - cur_class = rq->curr->sched_class; + cur_class = rq->donor->sched_class; preempt = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt); preempt_lazy = cpumask_test_cpu(cpu, pcpu->cpus_to_preempt_lazy); wait = cpumask_test_cpu(cpu, pcpu->cpus_to_wait); @@ -8921,7 +8945,7 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1); /* degrade to a plain, immediate kick */ immediate = true; - } else if (unlikely(!scx_set_task_slice(rq->curr, 0))) { + } else if (unlikely(!scx_set_task_slice(rq->donor, 0))) { __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1); } } @@ -9924,8 +9948,10 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, return false; /* - * Directly write only when we hold the lock of the rq @p is queued or - * running on. See the write rules above. + * Directly write only when we hold the lock of the rq @p is queued on or + * provides the current scheduling context for. Under proxy execution, + * rq->donor owns and consumes the slice while rq->curr executes on its + * behalf. See the slice write rules above. * * While @p is queued on a user DSQ or in the BPF scheduler, * synchronization is the scheduler's responsibility. This write can @@ -9939,7 +9965,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, locked_rq = scx_locked_rq(); if (!locked_rq || (READ_ONCE(p->scx.runnable_cpu) != cpu_of(locked_rq) && - !task_current(locked_rq, p))) { + !task_current_donor(locked_rq, p))) { set_task_slice_oob(sch, p, slice); return true; } @@ -10927,12 +10953,17 @@ __bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask) } /** - * scx_bpf_task_running - Is task currently running? + * scx_bpf_task_running - Is task the current scheduling context? * @p: task of interest + * + * Under proxy execution, this reports the donor rather than the task whose + * code is physically executing. The physical execution context is intentionally + * not exposed to the BPF scheduler, which continues to observe the donor as the + * running scheduling context. */ __bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p) { - return task_rq(p)->curr == p; + return rcu_access_pointer(task_rq(p)->donor) == p; } /** @@ -10993,10 +11024,15 @@ __bpf_kfunc struct rq *scx_bpf_locked_rq(const struct bpf_prog_aux *aux) } /** - * scx_bpf_cpu_curr - Return remote CPU's curr task + * scx_bpf_cpu_curr - Return remote CPU's current scheduling context * @cpu: CPU of interest * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs * + * Under proxy execution, this returns the donor, which supplies the scheduling + * policy and runtime budget, rather than the task whose code is physically + * executing. The physical execution context is intentionally not exposed to + * the BPF scheduler. + * * Callers must hold RCU read lock (KF_RCU). */ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_aux *aux) @@ -11012,7 +11048,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_ if (!scx_cpu_valid(sch, cpu, NULL)) return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); + return rcu_dereference(cpu_rq(cpu)->donor); } /** @@ -11036,7 +11072,7 @@ __bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_ cpu = scx_cid_to_cpu(sch, cid); if (cpu < 0) return NULL; - return rcu_dereference(cpu_rq(cpu)->curr); + return rcu_dereference(cpu_rq(cpu)->donor); } /** diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 08599bc57801..4541ef8eec95 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -285,7 +285,8 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) rq->scx.rescue.budget -= delta_exec; /* per-cpu usage average feeds the overload victim pick */ - pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq)); + pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu, + cpu_of(rq)); pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; if (!scx_rescue_slice_remaining(rq)) @@ -557,7 +558,7 @@ static void scx_rescue_timerfn(struct timer_list *timer) scx_rescue_admit(rq, p, slice); scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS, rq); - if (sched_class_above(&ext_sched_class, rq->curr->sched_class)) + if (sched_class_above(&ext_sched_class, rq->donor->sched_class)) resched_curr(rq); } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) { /* diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 7621c28c7e8c..c0174e523f2a 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -186,17 +186,18 @@ static inline u64 scx_caps_for_task(struct task_struct *p) return SCX_CAP_ENQ; } -/* the cap @sch needs to preempt @rq's current task, 0 if none */ -static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) +/* the cap @sch needs to preempt @rq's current scheduling context, 0 if none */ +static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, + u64 enq_flags) { - struct task_struct *curr = rq->curr; + struct task_struct *donor = rq->donor; /* a kernel-forced placement preempts regardless of caps */ if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS)) return 0; /* a non-ext task can't be preempted by ext, own-subtree needs no cap */ - if (curr->sched_class != &ext_sched_class || - scx_is_descendant(scx_task_sched(curr), sch)) + if (donor->sched_class != &ext_sched_class || + scx_is_descendant(scx_task_sched(donor), sch)) return 0; return SCX_CAP_PREEMPT; } From 74d9e0401539692198691d799a3b1ef809049528 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:51 +0200 Subject: [PATCH 37/69] sched_ext: Track proxy execution for NOHZ_FULL scx_can_stop_tick() currently treats any blocked rq->donor as an active proxy session. However, rq->donor can temporarily still identify an outgoing blocked task while the scheduler selects an ordinary execution context. A dependency update in that window can keep TICK_DEP_BIT_SCHED set after the context switch. Track whether proxy resolution actually selected a different execution context. Keep the tick active while that state is set instead of deriving it from a potentially stale donor. When proxy execution ends, clear the state and queue a balance callback to reevaluate the tick dependency after context_switch() updates rq->curr. This lets sched_can_stop_tick() observe the complete donor and execution selection without adding work outside the existing proxy-execution branch. Conservatively re-enable the periodic scheduler tick for the duration of proxy execution. Allowing proxy execution itself to run tickless would require making the remote NOHZ scheduler tick aware of the split between rq->curr and rq->donor, which is left as a future improvement. Perform the tick update from scx_proxy_reenqueue_retry(), which already runs after proxy resolution, so scheduler core needs no additional hook. Compile the additional runqueue state and callback out when CONFIG_NO_HZ_FULL is disabled. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 44 +++++++++++++++++++++++++++++++++++++++--- kernel/sched/sched.h | 4 ++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 9d41ec82f3bf..a7f23d881849 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1138,13 +1138,51 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +#ifdef CONFIG_NO_HZ_FULL +static void scx_proxy_tick_bal_cb(struct rq *rq) +{ + sched_update_tick_dependency(rq); +} + +static void scx_proxy_update_tick(struct rq *rq, struct task_struct *next) +{ + bool proxy = next != rq->donor; + bool was_proxy = rq->scx.flags & SCX_RQ_PROXY_TICK; + + if (likely(proxy == was_proxy)) + return; + + if (proxy) { + /* Keep proxy execution tick-driven for now. */ + rq->scx.flags |= SCX_RQ_PROXY_TICK; + tick_nohz_dep_set_cpu(cpu_of(rq), TICK_DEP_BIT_SCHED); + } else { + rq->scx.flags &= ~SCX_RQ_PROXY_TICK; + /* + * The selected donor is already visible, but rq->curr still + * identifies the outgoing execution context. Reevaluate after + * context_switch() updates rq->curr so sched_can_stop_tick() sees + * the complete selection. + */ + queue_balance_callback(rq, &rq->scx.proxy_tick_bal_cb, + scx_proxy_tick_bal_cb); + } +} +#endif + /* - * Retry proxy-rejected tasks which couldn't be reenqueued by an earlier drain. + * Complete sched_ext bookkeeping after proxy resolution and retry tasks which + * couldn't be reenqueued by an earlier reject DSQ drain. */ void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) { lockdep_assert_rq_held(rq); +#ifdef CONFIG_NO_HZ_FULL + if (scx_enabled() && tick_nohz_full_cpu(cpu_of(rq))) + scx_proxy_update_tick(rq, next); +#endif + if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; schedule_deferred_locked(rq); @@ -5049,8 +5087,8 @@ bool scx_can_stop_tick(struct rq *rq) struct task_struct *p = rq->donor; struct scx_sched *sch = scx_task_sched(p); - /* Keep the tick running while a blocked proxy donor is selected. */ - if (p->is_blocked) + /* Proxy execution is conservatively tick-driven for now. */ + if (rq->scx.flags & SCX_RQ_PROXY_TICK) return false; if (p->sched_class != &ext_sched_class) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 25e20f82788a..64a4ccc6e0d0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -792,6 +792,7 @@ enum scx_rq_flags { SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */ + SCX_RQ_PROXY_TICK = 1 << 10, /* proxy execution requires the tick */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_DISPATCH = 1 << 17, @@ -846,6 +847,9 @@ struct scx_rq { struct list_head deferred_reenq_users; /* user DSQs requesting reenq */ struct balance_callback deferred_bal_cb; struct balance_callback kick_sync_bal_cb; +#ifdef CONFIG_NO_HZ_FULL + struct balance_callback proxy_tick_bal_cb; +#endif struct irq_work deferred_irq_work; struct irq_work kick_cpus_irq_work; }; From ee172227d0dc9b169d41f5db154ed91db20073f7 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:52 +0200 Subject: [PATCH 38/69] sched_ext: Delegate proxy donor admission to BPF schedulers Proxy execution keeps a mutex-blocked donor runnable so that its scheduling context can execute the mutex owner. Define SCX_OPS_ENQ_BLOCKED as the admission contract for retained donors. Schedulers without the flag block EXT donors normally. Schedulers with the flag continue to own blocked donors and receive them through ops.enqueue() with SCX_ENQ_BLOCKED, allowing BPF to choose their DSQ, CPU and ordering. After BPF dispatches a donor, proxy execution may move its context to the rq of the mutex owner. This move is distinct from BPF placement: proxy_set_task_cpu() changes task_cpu() while preserving the donor's pre-proxy CPU in wake_cpu as the starting point for normal wakeup placement. A proxy-migrated donor returns through the full wakeup activation path when the mutex is released. Do not carry a retained proxy session across BPF scheduler ownership changes. Before root activation, parent-to-child takeover, rehome or punt, fully deactivate a retained donor unconditionally. The incoming scheduler then starts with clean task state and applies its admission policy the next time the task blocks. A retained donor can wake through ttwu_runnable() without another enqueue_task_scx() call. Use WF_TTWU_RQ to identify this path, where the wakeup skipped activation and enqueueing. wakeup_preempt_scx() requests dispatch reconsideration after is_blocked is cleared. A full wakeup activation has already enqueued the task and does not need another reschedule. Blocked-donor enqueueing takes precedence over the exiting and migration-disabled local-DSQ fallbacks, so that an opted-in scheduler sees every eligible donor request. Co-developed-by: John Stultz Signed-off-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 102 ++++++++++++++---- kernel/sched/ext/ext.h | 10 +- kernel/sched/ext/internal.h | 23 +++- kernel/sched/ext/sub.c | 12 ++- tools/sched_ext/include/scx/compat.h | 1 + .../sched_ext/include/scx/enum_defs.autogen.h | 1 + .../sched_ext/include/scx/enums.autogen.bpf.h | 3 + tools/sched_ext/include/scx/enums.autogen.h | 1 + .../sched_ext/include/scx/enums_abi.autogen.h | 2 +- 9 files changed, 131 insertions(+), 24 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index a7f23d881849..e48aac45afea 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -24,18 +24,24 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); -bool scx_allow_proxy_exec(const struct task_struct *p) +bool __scx_allow_proxy_exec(const struct task_struct *p) { - return p->sched_class != &ext_sched_class; + struct scx_sched *sch; + + if (p->sched_class != &ext_sched_class) + return true; + + sch = scx_task_sched(p); + return !sch || (sch->ops.flags & SCX_OPS_ENQ_BLOCKED); } /* - * End retained proxy execution before sched_ext takes ownership of @p. + * End retained proxy execution before changing @p's BPF scheduler ownership. * Called with @p's pi and rq locks held immediately before * sched_change_begin(). The caller must pass DEQUEUE_NOCLOCK so the rq clock * is updated only once. */ -static void scx_prepare_task_sched_change(struct task_struct *p) +void scx_prepare_task_sched_change(struct task_struct *p) { lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p)); @@ -2178,19 +2184,28 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct; - /* see %SCX_OPS_ENQ_EXITING */ - if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) && - unlikely(p->flags & PF_EXITING)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); - enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ - goto local; - } + /* + * A full wakeup enqueue happens before is_blocked is cleared. Don't + * report it as another blocked-donor admission. + */ + if ((sch->ops.flags & SCX_OPS_ENQ_BLOCKED) && p->is_blocked && + !(enq_flags & SCX_ENQ_WAKEUP)) { + enq_flags |= SCX_ENQ_BLOCKED; + } else { + /* see %SCX_OPS_ENQ_EXITING */ + if (unlikely(p->flags & PF_EXITING) && + !(sch->ops.flags & SCX_OPS_ENQ_EXITING)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1); + enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */ + goto local; + } - /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ - if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && - is_migration_disabled(p)) { - __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); - goto local; + /* see %SCX_OPS_ENQ_MIGRATION_DISABLED */ + if (!(sch->ops.flags & SCX_OPS_ENQ_MIGRATION_DISABLED) && + is_migration_disabled(p)) { + __scx_add_event(sch, SCX_EV_ENQ_SKIP_MIGRATION_DISABLED, 1); + goto local; + } } if (unlikely(!SCX_HAS_OP(sch, enqueue))) @@ -2532,10 +2547,23 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl /* * Preemption between SCX tasks is implemented by resetting the victim * task's slice to 0 and triggering reschedule on the target CPU. - * Nothing to do. + * + * A retained proxy donor can wake through ttwu_runnable() without another + * ops.enqueue(). WF_TTWU_RQ identifies this path. Request rescheduling so + * that ops.dispatch() can reconsider the task after ttwu_runnable() clears + * is_blocked. A full wakeup activation has already enqueued the task and + * doesn't need the additional reschedule. */ - if (p->sched_class == &ext_sched_class) + if (p->sched_class == &ext_sched_class) { + if (sched_proxy_exec() && (wake_flags & WF_TTWU_RQ) && + p->is_blocked) { + struct scx_sched *sch = scx_task_sched(p); + + if (sch && (sch->ops.flags & SCX_OPS_ENQ_BLOCKED)) + resched_curr(rq); + } return; + } /* * Getting preempted by a higher-priority class. Reenqueue IMMED tasks. @@ -2646,6 +2674,21 @@ static bool task_can_run_on_remote_rq(struct scx_sched *sch, WARN_ON_ONCE(task_cpu(p) == cpu); + /* + * proxy_set_task_cpu() preserves wake_cpu when moving a donor's scheduling + * context to its lock owner's rq. For example, if BPF places donor D on + * CPU0 while its lock owner runs on CPU1, proxy execution moves D to CPU1 + * but leaves D->wake_cpu pointing to CPU0. If D is later put on a shared + * DSQ, CPU0 could otherwise consume it and move it back, only for proxy + * execution to return it to CPU1 again. + * + * When task_cpu() == wake_cpu, the donor has not been proxy-migrated and + * BPF may still choose its placement. An actively donating task is + * rejected separately by task_proxy_running_or_donating(). + */ + if (sched_proxy_exec() && p->is_blocked && task_cpu(p) != p->wake_cpu) + return false; + /* * If @p has migration disabled, @p->cpus_ptr is updated to contain only * the pinned CPU in migrate_disable_switch() while @p is being switched @@ -3469,6 +3512,24 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (p->scx.flags & SCX_TASK_QUEUED) { set_task_runnable(rq, p); + /* Delegate retained donor admission to its owning BPF scheduler. */ + if (p->is_blocked) { + /* + * If the donor is the same and only the mutex owner + * changes, avoid triggering another ops.enqueue(): the + * BPF scheduler has already admitted the donor, so it + * can continue running. + */ + if (next == p) + goto switch_class; + + if (WARN_ON_ONCE(!sch)) + goto switch_class; + WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_BLOCKED)); + scx_do_enqueue_task(rq, p, 0, -1); + goto switch_class; + } + /* * If @p has slice left and is being put, @p is getting * preempted by a higher priority scheduler class or core-sched @@ -7904,6 +7965,11 @@ int scx_validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops) return -EINVAL; } + if ((ops->flags & SCX_OPS_ENQ_BLOCKED) && !ops->enqueue) { + scx_error(sch, "SCX_OPS_ENQ_BLOCKED requires ops.enqueue() to be implemented"); + return -EINVAL; + } + /* * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched * may set it to declare a dependency; reject if the root hasn't diff --git a/kernel/sched/ext/ext.h b/kernel/sched/ext/ext.h index 8348d6008651..d32bc01071b0 100644 --- a/kernel/sched/ext/ext.h +++ b/kernel/sched/ext/ext.h @@ -21,12 +21,20 @@ int scx_check_setscheduler(struct task_struct *p, int policy); void scx_prepare_setscheduler(struct task_struct *p, int policy); bool task_should_scx(int policy); bool scx_allow_ttwu_queue(const struct task_struct *p); -bool scx_allow_proxy_exec(const struct task_struct *p); +bool __scx_allow_proxy_exec(const struct task_struct *p); void scx_proxy_donor_start(struct rq *rq); void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next); void init_sched_ext_class(void); void __scx_update_idle(struct rq *rq, bool idle, bool do_notify); +static inline bool scx_allow_proxy_exec(const struct task_struct *p) +{ + if (scx_enabled()) + return __scx_allow_proxy_exec(p); + + return true; +} + static inline void scx_update_idle(struct rq *rq, bool idle, bool do_notify) { if (scx_enabled()) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index deb1a2b8ab9b..d65fec631bdf 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -230,6 +230,19 @@ enum scx_ops_flags { */ SCX_OPS_LAZY_RESCHED = 1LLU << 9, + /* + * If set, mutex-blocked tasks remain runnable as proxy donors and are + * passed to ops.enqueue() with %SCX_ENQ_BLOCKED. The BPF scheduler controls + * when donors are dispatched and whether they should preempt other work. + * + * If clear, mutex-blocked tasks are removed from the runqueue normally + * and cannot donate their scheduling context through proxy execution. + * + * For blocked donors, this flag takes precedence over + * %SCX_OPS_ENQ_EXITING and %SCX_OPS_ENQ_MIGRATION_DISABLED. + */ + SCX_OPS_ENQ_BLOCKED = 1LLU << 10, + SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE | SCX_OPS_ENQ_LAST | SCX_OPS_ENQ_EXITING | @@ -239,7 +252,8 @@ enum scx_ops_flags { SCX_OPS_BUILTIN_IDLE_PER_NODE | SCX_OPS_ALWAYS_ENQ_IMMED | SCX_OPS_TID_TO_TASK | - SCX_OPS_LAZY_RESCHED, + SCX_OPS_LAZY_RESCHED | + SCX_OPS_ENQ_BLOCKED, /* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */ __SCX_OPS_INTERNAL_MASK = 0xffLLU << 56, @@ -1812,6 +1826,12 @@ enum scx_enq_flags { */ SCX_ENQ_LAST = 1LLU << 41, + /* + * The task is blocked on a mutex and is being kept runnable as a proxy + * donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set. + */ + SCX_ENQ_BLOCKED = 1LLU << 42, + /* high 8 bits are internal */ __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56, @@ -2152,6 +2172,7 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter); bool scx_set_task_slice(struct task_struct *p, u64 slice); void scx_task_slice_ended(struct rq *rq, struct task_struct *p); void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq); +void scx_prepare_task_sched_change(struct task_struct *p); void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p); void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, int sticky_cpu); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 4541ef8eec95..9af09807951d 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1207,7 +1207,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) lockdep_assert_held(&p->pi_lock); lockdep_assert_rq_held(task_rq(p)); - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_state(p, SCX_TASK_INIT_BEGIN); scx_set_task_state(p, SCX_TASK_INIT); @@ -1237,7 +1239,9 @@ static void scx_punt_task(struct scx_sched *to, struct task_struct *p) lockdep_assert_rq_held(task_rq(p)); WARN_ON_ONCE(!READ_ONCE(to->bypass_depth)); - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { scx_disable_and_exit_task(scx_task_sched(p), p); scx_set_task_sched(p, to); } @@ -1895,7 +1899,9 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (!(p->scx.flags & SCX_TASK_SUB_INIT)) continue; - scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { + scx_prepare_task_sched_change(p); + scoped_guard (sched_change, p, + DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK) { /* * $p must be either READY or ENABLED. If ENABLED, * __scx_disabled_and_exit_task() first disables and diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index d4beb7a7c68a..58253fe8e85d 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -214,6 +214,7 @@ static inline bool __COMPAT_struct_has_field(const char *type, const char *field #define SCX_OPS_ALLOW_QUEUED_WAKEUP SCX_OPS_FLAG(SCX_OPS_ALLOW_QUEUED_WAKEUP) #define SCX_OPS_BUILTIN_IDLE_PER_NODE SCX_OPS_FLAG(SCX_OPS_BUILTIN_IDLE_PER_NODE) #define SCX_OPS_ALWAYS_ENQ_IMMED SCX_OPS_FLAG(SCX_OPS_ALWAYS_ENQ_IMMED) +#define SCX_OPS_ENQ_BLOCKED SCX_OPS_FLAG(SCX_OPS_ENQ_BLOCKED) #define SCX_PICK_IDLE_FLAG(name) __COMPAT_ENUM_OR_ZERO("scx_pick_idle_cpu_flags", #name) diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 015eb6f40068..9707abeee00d 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -90,6 +90,7 @@ #define HAVE_SCX_ENQ_RESCUE #define HAVE_SCX_ENQ_REENQ #define HAVE_SCX_ENQ_LAST +#define HAVE_SCX_ENQ_BLOCKED #define HAVE___SCX_ENQ_INTERNAL_MASK #define HAVE_SCX_ENQ_CLEAR_OPSS #define HAVE_SCX_ENQ_DSQ_PRIQ diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index 21e1a79d5ecd..0a5478afbe79 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -142,6 +142,9 @@ const volatile u64 __SCX_ENQ_REENQ __weak; const volatile u64 __SCX_ENQ_LAST __weak; #define SCX_ENQ_LAST __SCX_ENQ_LAST +const volatile u64 __SCX_ENQ_BLOCKED __weak; +#define SCX_ENQ_BLOCKED __SCX_ENQ_BLOCKED + const volatile u64 __SCX_ENQ_CLEAR_OPSS __weak; #define SCX_ENQ_CLEAR_OPSS __SCX_ENQ_CLEAR_OPSS diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index 6e0f6a3a219a..60a1e5576126 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -51,6 +51,7 @@ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \ + SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_BLOCKED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_DSQ_PRIQ); \ SCX_ENUM_SET(skel, scx_deq_flags, SCX_DEQ_SCHED_CHANGE); \ diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index f1bb6d70bf1d..251eea9b81e7 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -180,7 +180,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU }, { "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU }, { "scx_ops_flags", "SCX_OPS_LAZY_RESCHED", 0x200LLU }, - { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x3ffLLU }, + { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x7ffLLU }, { "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU }, { "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU }, { "scx_ops_state", "SCX_OPSS_NONE", 0x0LLU }, From f73ec679142421236e082e79650765c1ad44a47c Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:53 +0200 Subject: [PATCH 39/69] sched_ext: Add selftest for blocked donor admission SCX_OPS_ENQ_BLOCKED allows BPF schedulers to receive blocked proxy donors through ops.enqueue(). SCX_ENQ_BLOCKED identifies blocked-donor admission requests. Add selftest coverage for this interface. Exercise a priority inversion using a weighted-vruntime BPF scheduler. A nice +19 owner holds a shared mutex, a nice -20 donor blocks on it, and nice 0 CPU contenders, one per allowed CPU, keep the system busy. Test both a same-CPU topology and a cross-CPU topology with the donor and owner on different CPUs. Treat blocked donors according to the normal BPF ordering policy and assign the default slice on every enqueue, as for other tasks. Run each CPU placement configuration with SCX_OPS_ENQ_BLOCKED first disabled and then enabled, count blocked-donor enqueues by CPU and report average mutex hold and wait times. Verify that full wakeups are never reported as blocked-donor admissions by checking that SCX_ENQ_WAKEUP and SCX_ENQ_BLOCKED are not set together. Proxy execution coverage requires CONFIG_SCHED_PROXY_EXEC=y, which the selftest config selects. Access to the kernel mutex is provided via a loadable kernel module, built through TEST_GEN_MODS_DIR and managed by the test. Acked-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/.gitignore | 4 + tools/testing/selftests/sched_ext/Makefile | 2 + tools/testing/selftests/sched_ext/config | 2 + .../selftests/sched_ext/enq_blocked.bpf.c | 116 +++ .../testing/selftests/sched_ext/enq_blocked.c | 917 ++++++++++++++++++ .../testing/selftests/sched_ext/enq_blocked.h | 28 + .../selftests/sched_ext/test_modules/Makefile | 13 + .../test_modules/scx_enq_blocked_test.c | 195 ++++ 8 files changed, 1277 insertions(+) create mode 100644 tools/testing/selftests/sched_ext/enq_blocked.bpf.c create mode 100644 tools/testing/selftests/sched_ext/enq_blocked.c create mode 100644 tools/testing/selftests/sched_ext/enq_blocked.h create mode 100644 tools/testing/selftests/sched_ext/test_modules/Makefile create mode 100644 tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c diff --git a/tools/testing/selftests/sched_ext/.gitignore b/tools/testing/selftests/sched_ext/.gitignore index ae5491a114c0..54a1fd2af713 100644 --- a/tools/testing/selftests/sched_ext/.gitignore +++ b/tools/testing/selftests/sched_ext/.gitignore @@ -4,3 +4,7 @@ !Makefile !.gitignore !config +!test_modules/ +!test_modules/scx_enq_blocked_test.c +!test_modules/Makefile +test_modules/*.mod.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index 68493e49f47a..286b510fd76e 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -5,6 +5,7 @@ include ../../../scripts/Makefile.arch include ../../../scripts/Makefile.include TEST_GEN_PROGS := runner +TEST_GEN_MODS_DIR := test_modules # override lib.mk's default rules OVERRIDE_TARGETS := 1 @@ -165,6 +166,7 @@ auto-test-targets := \ create_dsq \ dequeue \ dequeue_iter \ + enq_blocked \ enq_last_no_enq_fails \ ddsp_bogus_dsq_fail \ ddsp_vtimelocal_fail \ diff --git a/tools/testing/selftests/sched_ext/config b/tools/testing/selftests/sched_ext/config index aa901b05c8ad..affa3cf33470 100644 --- a/tools/testing/selftests/sched_ext/config +++ b/tools/testing/selftests/sched_ext/config @@ -6,3 +6,5 @@ CONFIG_BPF=y CONFIG_BPF_SYSCALL=y CONFIG_DEBUG_INFO=y CONFIG_DEBUG_INFO_BTF=y +CONFIG_EXPERT=y +CONFIG_SCHED_PROXY_EXEC=y diff --git a/tools/testing/selftests/sched_ext/enq_blocked.bpf.c b/tools/testing/selftests/sched_ext/enq_blocked.bpf.c new file mode 100644 index 000000000000..212690bf4e07 --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.bpf.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Verify that SCX_OPS_ENQ_BLOCKED passes blocked proxy donors through + * ops.enqueue() and record whether callbacks occur on the donor or owner CPU. + */ + +#include + +#define SHARED_DSQ 0 + +char _license[] SEC("license") = "GPL"; + +s32 donor_pid; +s32 donor_cpu = -1; +s32 owner_cpu = -1; +u64 nr_blocked_enqueues; +u64 nr_blocked_enqueues_donor_cpu; +u64 nr_blocked_enqueues_owner_cpu; +u64 nr_blocked_enqueues_other_cpu; +u64 nr_blocked_wakeups; +static u64 vtime_now; + +UEI_DEFINE(uei); + +s32 BPF_STRUCT_OPS(enq_blocked_select_cpu, + struct task_struct *p, s32 prev_cpu, u64 wake_flags) +{ + return prev_cpu; +} + +void BPF_STRUCT_OPS(enq_blocked_enqueue, struct task_struct *p, u64 enq_flags) +{ + u64 vtime = p->scx.dsq_vtime; + + if (enq_flags & SCX_ENQ_BLOCKED) { + int cpu = scx_bpf_task_cpu(p); + + if (enq_flags & SCX_ENQ_WAKEUP) + __sync_fetch_and_add(&nr_blocked_wakeups, 1); + + if (p->pid == donor_pid) { + __sync_fetch_and_add(&nr_blocked_enqueues, 1); + if (cpu == donor_cpu) + __sync_fetch_and_add(&nr_blocked_enqueues_donor_cpu, 1); + else if (cpu == owner_cpu) + __sync_fetch_and_add(&nr_blocked_enqueues_owner_cpu, 1); + else + __sync_fetch_and_add(&nr_blocked_enqueues_other_cpu, 1); + } + } + + /* Limit the amount of budget an idling task can accumulate. */ + if (time_before(vtime, vtime_now - SCX_SLICE_DFL)) + vtime = vtime_now - SCX_SLICE_DFL; + + scx_bpf_dsq_insert_vtime(p, SHARED_DSQ, SCX_SLICE_DFL, vtime, + enq_flags); + scx_bpf_kick_cpu(scx_bpf_task_cpu(p), SCX_KICK_IDLE); +} + +void BPF_STRUCT_OPS(enq_blocked_dispatch, s32 cpu, struct task_struct *prev) +{ + scx_bpf_dsq_move_to_local(SHARED_DSQ, 0); +} + +void BPF_STRUCT_OPS(enq_blocked_running, struct task_struct *p) +{ + if (time_before(vtime_now, p->scx.dsq_vtime)) + vtime_now = p->scx.dsq_vtime; +} + +void BPF_STRUCT_OPS(enq_blocked_stopping, struct task_struct *p, bool runnable) +{ + u64 delta = scale_by_task_weight_inverse(p, + SCX_SLICE_DFL - p->scx.slice); + + scx_bpf_task_set_dsq_vtime(p, p->scx.dsq_vtime + delta); +} + +void BPF_STRUCT_OPS(enq_blocked_enable, struct task_struct *p) +{ + scx_bpf_task_set_dsq_vtime(p, vtime_now); +} + +s32 BPF_STRUCT_OPS_SLEEPABLE(enq_blocked_init) +{ + int ret; + + ret = scx_bpf_create_dsq(SHARED_DSQ, -1); + if (ret) { + scx_bpf_error("failed to create DSQ %d (%d)", SHARED_DSQ, ret); + return ret; + } + + return 0; +} + +void BPF_STRUCT_OPS(enq_blocked_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops enq_blocked_ops = { + .select_cpu = (void *)enq_blocked_select_cpu, + .enqueue = (void *)enq_blocked_enqueue, + .dispatch = (void *)enq_blocked_dispatch, + .running = (void *)enq_blocked_running, + .stopping = (void *)enq_blocked_stopping, + .enable = (void *)enq_blocked_enable, + .init = (void *)enq_blocked_init, + .exit = (void *)enq_blocked_exit, + .name = "enq_blocked", +}; diff --git a/tools/testing/selftests/sched_ext/enq_blocked.c b/tools/testing/selftests/sched_ext/enq_blocked.c new file mode 100644 index 000000000000..26204548bbf0 --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.c @@ -0,0 +1,917 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Exercise a priority inversion with the owner and donor first pinned to the + * same CPU, then with each on a different CPU. A high-priority donor blocks on + * a mutex held by a low-priority owner while one medium-priority contender per + * available CPU keeps the system busy. A weighted-vruntime BPF scheduler runs + * both CPU placement configurations with SCX_OPS_ENQ_BLOCKED first disabled + * and then enabled. The test validates blocked-donor admission and reports the + * average mutex hold and wait times, plus their enabled-minus-disabled deltas, + * for each configuration. The timing data is informational. + * + * CONFIG_SCHED_PROXY_EXEC=y is required to exercise the proxy-execution paths. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "enq_blocked.bpf.skel.h" +#include "enq_blocked.h" +#include "scx_test.h" + +#define MODULE_NAME "scx_enq_blocked_test" +#define MODULE_FILE "test_modules/" MODULE_NAME ".ko" +#define DEVICE_PATH "/dev/scx_enq_blocked" +#define WAIT_STEP_US 1000 +#define WAIT_TIMEOUT_MS 2000 +#define NR_WARMUP_TRIALS 1 +#define NR_MEASURED_TRIALS 10 +#define NR_TRIALS (NR_WARMUP_TRIALS + NR_MEASURED_TRIALS) +#define JOIN_TIMEOUT_MS ((NR_TRIALS + 1) * WAIT_TIMEOUT_MS) +#define OWNER_NICE 19 +#define DONOR_NICE -20 +#define CONTENDER_NICE 0 + +struct thread_ctx { + atomic_bool start_donor; + atomic_bool abort; + atomic_bool stop_contender; + atomic_bool measurement_ready; + atomic_int donor_pid; + atomic_int donor_completed; + int fd; + int donor_cpu; + int owner_cpu; +}; + +struct contender_ctx { + struct thread_ctx *thread_ctx; + atomic_int status; + int cpu; +}; + +struct run_result { + struct enq_blocked_stats stats; + u64 nr_blocked_enqueues; + u64 nr_blocked_enqueues_donor_cpu; + u64 nr_blocked_enqueues_owner_cpu; + u64 nr_blocked_enqueues_other_cpu; + u64 nr_blocked_wakeups; +}; + +static bool parse_bool(const char *value, bool *result) +{ + if (!strcasecmp(value, "1") || !strcasecmp(value, "y") || + !strcasecmp(value, "yes") || !strcasecmp(value, "on") || + !strcasecmp(value, "true")) { + *result = true; + return true; + } + + if (!strcasecmp(value, "0") || !strcasecmp(value, "n") || + !strcasecmp(value, "no") || !strcasecmp(value, "off") || + !strcasecmp(value, "false")) { + *result = false; + return true; + } + + return false; +} + +static bool cmdline_bool(const char *name, bool default_value) +{ + char cmdline[4096], *newline, *saveptr = NULL, *token; + size_t name_len = strlen(name); + bool value = default_value; + FILE *file; + + file = fopen("/proc/cmdline", "r"); + if (!file) + return default_value; + + if (!fgets(cmdline, sizeof(cmdline), file)) { + fclose(file); + return default_value; + } + fclose(file); + newline = strchr(cmdline, '\n'); + if (newline) + *newline = '\0'; + + for (token = strtok_r(cmdline, " ", &saveptr); token; + token = strtok_r(NULL, " ", &saveptr)) { + bool parsed; + + if (strncmp(token, name, name_len) || token[name_len] != '=') + continue; + if (parse_bool(token + name_len + 1, &parsed)) + value = parsed; + } + + return value; +} + +static int module_path(char *path, size_t size) +{ + ssize_t len; + char *slash; + + len = readlink("/proc/self/exe", path, size - 1); + if (len < 0) + return -errno; + path[len] = '\0'; + + slash = strrchr(path, '/'); + if (!slash) + return -EINVAL; + *slash = '\0'; + + if (snprintf(slash, size - (slash - path), "/%s", MODULE_FILE) >= + size - (slash - path)) + return -ENAMETOOLONG; + + return 0; +} + +static int load_test_module(bool *loaded_here) +{ + char path[PATH_MAX]; + int fd, err; + + err = module_path(path, sizeof(path)); + if (err) + return err; + + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -errno; + + if (syscall(SYS_finit_module, fd, "", 0)) { + err = errno; + close(fd); + if (err == EEXIST) + return 0; + return -err; + } + + close(fd); + *loaded_here = true; + return 0; +} + +static void unload_test_module(bool loaded_here) +{ + if (loaded_here && syscall(SYS_delete_module, MODULE_NAME, O_NONBLOCK)) + SCX_ERR("Failed to unload %s (%d)", MODULE_NAME, errno); +} + +static int pin_to_cpu(int cpu) +{ + cpu_set_t mask; + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + return sched_setaffinity(0, sizeof(mask), &mask) ? errno : 0; +} + +static int select_test_cpus(bool cross_cpu, cpu_set_t *mask, int *donor_cpu, + int *owner_cpu) +{ + int cpu, first = -1; + + if (sched_getaffinity(0, sizeof(*mask), mask)) + return -errno; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, mask)) + continue; + if (first < 0) { + first = cpu; + if (!cross_cpu) + break; + } else { + *donor_cpu = first; + *owner_cpu = cpu; + return 0; + } + } + + if (first < 0) + return -ENODEV; + if (cross_cpu) + return -EAGAIN; + + *donor_cpu = first; + *owner_cpu = first; + return 0; +} + +static int set_nice(int nice) +{ + return setpriority(PRIO_PROCESS, 0, nice) ? errno : 0; +} + +static bool wait_for_pid(atomic_int *pid) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (atomic_load_explicit(pid, memory_order_acquire) > 0) + return true; + usleep(WAIT_STEP_US); + } + + return false; +} + +static int wait_for_contenders(struct contender_ctx *contenders, + size_t nr_contenders) +{ + size_t i, nr_ready; + int status, waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + nr_ready = 0; + for (i = 0; i < nr_contenders; i++) { + status = atomic_load_explicit(&contenders[i].status, + memory_order_acquire); + if (status < 0) + return status; + if (status > 0) + nr_ready++; + } + if (nr_ready == nr_contenders) + return 1; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static int wait_for_donor_state(struct thread_ctx *ctx, int expected) +{ + int state, waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + state = ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_DONOR_STATE); + if (state == expected) + return state; + if (state < 0 && errno != ENOENT) + return -errno; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static bool wait_for_donor(struct thread_ctx *ctx, int trial) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (atomic_load_explicit(&ctx->donor_completed, + memory_order_acquire) >= trial) + return true; + if (atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + return false; + usleep(WAIT_STEP_US); + } + + return false; +} + +static bool wait_for_measurement(struct thread_ctx *ctx) +{ + while (!atomic_load_explicit(&ctx->measurement_ready, + memory_order_acquire) && + !atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + sched_yield(); + + return !atomic_load_explicit(&ctx->abort, memory_order_relaxed); +} + +static void *contender_fn(void *arg) +{ + struct contender_ctx *contender = arg; + struct thread_ctx *ctx = contender->thread_ctx; + int err; + + err = pin_to_cpu(contender->cpu); + if (!err) + err = set_nice(CONTENDER_NICE); + atomic_store_explicit(&contender->status, err ? -err : 1, + memory_order_release); + if (err) + return (void *)(uintptr_t)err; + + while (!atomic_load_explicit(&ctx->stop_contender, + memory_order_relaxed)) + ; + + return NULL; +} + +static void *owner_fn(void *arg) +{ + struct thread_ctx *ctx = arg; + int err, i; + + err = pin_to_cpu(ctx->owner_cpu); + if (err) + return (void *)(uintptr_t)err; + err = set_nice(OWNER_NICE); + if (err) + return (void *)(uintptr_t)err; + + for (i = 0; i < NR_TRIALS; i++) { + if (ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_OWNER)) + return (void *)(uintptr_t)errno; + if (!wait_for_donor(ctx, i + 1)) + return (void *)(uintptr_t)ETIMEDOUT; + + if (i + 1 == NR_WARMUP_TRIALS && !wait_for_measurement(ctx)) + return NULL; + } + + return NULL; +} + +static int run_donor_trial(struct thread_ctx *ctx) +{ + int waited_ms; + + for (waited_ms = 0; waited_ms < WAIT_TIMEOUT_MS; waited_ms++) { + if (!ioctl(ctx->fd, ENQ_BLOCKED_IOCTL_DONOR)) + return 0; + if (errno != EAGAIN) + return -errno; + usleep(WAIT_STEP_US); + } + + return -ETIMEDOUT; +} + +static void *donor_fn(void *arg) +{ + struct thread_ctx *ctx = arg; + int err, i; + + err = pin_to_cpu(ctx->donor_cpu); + if (err) + return (void *)(uintptr_t)err; + err = set_nice(DONOR_NICE); + if (err) + return (void *)(uintptr_t)err; + + atomic_store_explicit(&ctx->donor_pid, syscall(SYS_gettid), + memory_order_release); + while (!atomic_load_explicit(&ctx->start_donor, memory_order_acquire) && + !atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + sched_yield(); + + if (atomic_load_explicit(&ctx->abort, memory_order_relaxed)) + return NULL; + + for (i = 0; i < NR_TRIALS; i++) { + err = run_donor_trial(ctx); + if (err) + return (void *)(uintptr_t)-err; + atomic_store_explicit(&ctx->donor_completed, i + 1, + memory_order_release); + } + + return NULL; +} + +static void print_avg_time(const char *name, u64 total_ns, u64 samples) +{ + u64 avg_ns = samples ? total_ns / samples : 0; + + printf(" %s_avg_ns=%llu (%llu.%03llu ms, samples=%llu)\n", name, + (unsigned long long)avg_ns, + (unsigned long long)(avg_ns / 1000000), + (unsigned long long)((avg_ns / 1000) % 1000), + (unsigned long long)samples); +} + +static void print_avg_delta(const char *name, u64 disabled_total, + u64 disabled_samples, u64 enabled_total, + u64 enabled_samples) +{ + u64 disabled_avg, enabled_avg; + s64 delta_ns; + double delta_pct; + + if (!disabled_samples || !enabled_samples) + return; + + disabled_avg = disabled_total / disabled_samples; + enabled_avg = enabled_total / enabled_samples; + delta_ns = (s64)enabled_avg - (s64)disabled_avg; + delta_pct = disabled_avg ? 100.0 * delta_ns / disabled_avg : 0.0; + + printf(" %s_delta_ns=%+lld (%+.2f%%)\n", name, + (long long)delta_ns, delta_pct); +} + +static int join_thread(pthread_t thread, const struct timespec *deadline, + int *thread_err) +{ + void *result; + int err; + + err = pthread_timedjoin_np(thread, &result, deadline); + if (err) + return err; + + *thread_err = (int)(uintptr_t)result; + return 0; +} + +static void set_join_deadline(struct timespec *deadline) +{ + clock_gettime(CLOCK_REALTIME, deadline); + deadline->tv_sec += JOIN_TIMEOUT_MS / 1000; + deadline->tv_nsec += (JOIN_TIMEOUT_MS % 1000) * 1000000; + if (deadline->tv_nsec >= 1000000000) { + deadline->tv_sec++; + deadline->tv_nsec -= 1000000000; + } +} + +static enum scx_test_status setup(void **ctx) +{ + struct enq_blocked *skel; + u64 flag; + + skel = enq_blocked__open(); + SCX_FAIL_IF(!skel, "Failed to open skel"); + SCX_ENUM_INIT(skel); + + flag = SCX_OPS_ENQ_BLOCKED; + if (!flag) { + enq_blocked__destroy(skel); + fprintf(stderr, "SKIP: SCX_OPS_ENQ_BLOCKED is unavailable\n"); + return SCX_TEST_SKIP; + } + + enq_blocked__destroy(skel); + *ctx = NULL; + return SCX_TEST_PASS; +} + +static enum scx_test_status run_one(bool enq_blocked, bool cross_cpu, + struct run_result *result) +{ + struct enq_blocked *skel; + struct thread_ctx thread_ctx = {}; + struct contender_ctx *contender_ctxs = NULL; + struct bpf_link *link = NULL; + pthread_t owner, donor, *contenders = NULL; + struct timespec join_deadline; + cpu_set_t allowed_cpus; + bool module_loaded = false; + bool owner_started = false, donor_started = false; + bool join_timed_out = false; + bool proxy_enabled; + enum scx_test_status status = SCX_TEST_PASS; + int cpu, donor_pid, donor_state, err, thread_err; + size_t i, nr_contenders, nr_contenders_started = 0; + size_t nr_contenders_joined = 0; + u64 nr_blocked, nr_blocked_donor_cpu, nr_blocked_owner_cpu; + u64 nr_blocked_other_cpu, nr_blocked_wakeups; + struct enq_blocked_stats stats; + + err = select_test_cpus(cross_cpu, &allowed_cpus, &thread_ctx.donor_cpu, + &thread_ctx.owner_cpu); + if (err == -EAGAIN) { + fprintf(stderr, "SKIP: cross-CPU case requires two allowed CPUs\n"); + return SCX_TEST_SKIP; + } + if (err) { + SCX_ERR("Failed to select test CPUs (%d)", -err); + return SCX_TEST_FAIL; + } + nr_contenders = CPU_COUNT(&allowed_cpus); + contenders = calloc(nr_contenders, sizeof(*contenders)); + contender_ctxs = calloc(nr_contenders, sizeof(*contender_ctxs)); + if (!contenders || !contender_ctxs) { + SCX_ERR("Failed to allocate %zu contender threads", nr_contenders); + status = SCX_TEST_FAIL; + goto out_contenders; + } + + skel = enq_blocked__open(); + if (!skel) { + SCX_ERR("Failed to open skel"); + status = SCX_TEST_FAIL; + goto out_contenders; + } + SCX_ENUM_INIT(skel); + skel->struct_ops.enq_blocked_ops->flags = + SCX_OPS_ENQ_LAST | + (enq_blocked ? SCX_OPS_ENQ_BLOCKED : 0); + if (enq_blocked__load(skel)) { + SCX_ERR("Failed to load skel"); + status = SCX_TEST_FAIL; + goto out_skel; + } + + err = load_test_module(&module_loaded); + if (err == -EPERM || err == -ENOENT) { + fprintf(stderr, "SKIP: cannot load mutex fixture (%d)\n", -err); + status = SCX_TEST_SKIP; + goto out_skel; + } + if (err) { + SCX_ERR("Failed to load mutex fixture (%d)", -err); + status = SCX_TEST_FAIL; + goto out_skel; + } + + thread_ctx.fd = open(DEVICE_PATH, O_RDONLY | O_CLOEXEC); + if (thread_ctx.fd < 0) { + SCX_ERR("Failed to open %s (%d)", DEVICE_PATH, errno); + status = SCX_TEST_FAIL; + goto out_module; + } + err = ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED); + if (err < 0) { + SCX_ERR("Failed to query proxy-exec support (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + proxy_enabled = err && cmdline_bool("sched_proxy_exec", true); + if (!proxy_enabled) { + fprintf(stderr, "SKIP: proxy execution is not enabled\n"); + status = SCX_TEST_SKIP; + goto out_fd; + } + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_RESET_STATS)) { + SCX_ERR("Failed to reset mutex statistics (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_PREP_ATTACH)) { + SCX_ERR("Failed to prepare scheduler attachment (%d)", errno); + status = SCX_TEST_FAIL; + goto out_fd; + } + + err = pthread_create(&owner, NULL, owner_fn, &thread_ctx); + if (err) { + SCX_ERR("Failed to create owner thread (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } + owner_started = true; + + err = pthread_create(&donor, NULL, donor_fn, &thread_ctx); + if (err) { + SCX_ERR("Failed to create donor thread (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } + donor_started = true; + + if (!wait_for_pid(&thread_ctx.donor_pid)) { + SCX_ERR("Timed out waiting for donor thread"); + status = SCX_TEST_FAIL; + goto out; + } + + donor_pid = atomic_load_explicit(&thread_ctx.donor_pid, + memory_order_acquire); + skel->bss->donor_pid = donor_pid; + skel->data->donor_cpu = thread_ctx.donor_cpu; + skel->data->owner_cpu = thread_ctx.owner_cpu; + atomic_store_explicit(&thread_ctx.start_donor, true, + memory_order_release); + + donor_state = ENQ_BLOCKED_DONOR_SLEEPING; + if (proxy_enabled) + donor_state |= ENQ_BLOCKED_DONOR_ON_RQ; + err = wait_for_donor_state(&thread_ctx, donor_state); + if (err < 0) { + SCX_ERR("Donor did not block before scheduler attachment (%d)", -err); + status = SCX_TEST_FAIL; + goto out; + } + + link = bpf_map__attach_struct_ops(skel->maps.enq_blocked_ops); + if (!link) { + SCX_ERR("Failed to attach scheduler"); + status = SCX_TEST_FAIL; + goto out; + } + + /* Scheduler ownership changes start from a fully blocked donor. */ + donor_state = ENQ_BLOCKED_DONOR_SLEEPING; + err = wait_for_donor_state(&thread_ctx, donor_state); + if (err < 0) { + SCX_ERR("Unexpected donor state after scheduler attachment (%d)", + -err); + status = SCX_TEST_FAIL; + goto out; + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_ATTACH_DONE)) { + SCX_ERR("Failed to complete scheduler attachment (%d)", errno); + status = SCX_TEST_FAIL; + goto out; + } + + i = 0; + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &allowed_cpus)) + continue; + + contender_ctxs[i].thread_ctx = &thread_ctx; + contender_ctxs[i].cpu = cpu; + atomic_init(&contender_ctxs[i].status, 0); + err = pthread_create(&contenders[i], NULL, contender_fn, + &contender_ctxs[i]); + if (err) { + SCX_ERR("Failed to create contender for CPU %d (%d)", + cpu, err); + status = SCX_TEST_FAIL; + goto out; + } + nr_contenders_started++; + i++; + } + + err = wait_for_contenders(contender_ctxs, nr_contenders); + if (err != 1) { + SCX_ERR("Contender threads failed (%d)", -err); + status = SCX_TEST_FAIL; + goto out; + } + + /* + * The first trial spans scheduler attachment and validates the state + * transition, but including it would skew scheduling latency. Exclude it + * from both the mutex and BPF enqueue measurements. + */ + if (!wait_for_donor(&thread_ctx, NR_WARMUP_TRIALS)) { + SCX_ERR("Timed out waiting for warm-up trial"); + status = SCX_TEST_FAIL; + goto out; + } + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_RESET_STATS)) { + SCX_ERR("Failed to reset mutex statistics after warm-up (%d)", + errno); + status = SCX_TEST_FAIL; + goto out; + } + skel->bss->nr_blocked_enqueues = 0; + skel->bss->nr_blocked_enqueues_donor_cpu = 0; + skel->bss->nr_blocked_enqueues_owner_cpu = 0; + skel->bss->nr_blocked_enqueues_other_cpu = 0; + skel->bss->nr_blocked_wakeups = 0; + atomic_store_explicit(&thread_ctx.measurement_ready, true, + memory_order_release); + +out: + ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_ATTACH_DONE); + if (status != SCX_TEST_PASS) { + atomic_store_explicit(&thread_ctx.abort, true, memory_order_release); + atomic_store_explicit(&thread_ctx.start_donor, true, + memory_order_release); + atomic_store_explicit(&thread_ctx.measurement_ready, true, + memory_order_release); + } + + set_join_deadline(&join_deadline); + if (donor_started) { + err = join_thread(donor, &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for donor thread"); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join donor thread (%d)", err); + status = SCX_TEST_FAIL; + } else { + donor_started = false; + if (thread_err) { + SCX_ERR("Donor thread failed (%d)", thread_err); + status = SCX_TEST_FAIL; + } + } + } + if (!join_timed_out && owner_started) { + err = join_thread(owner, &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for owner thread"); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join owner thread (%d)", err); + status = SCX_TEST_FAIL; + } else { + owner_started = false; + if (thread_err) { + SCX_ERR("Owner thread failed (%d)", thread_err); + status = SCX_TEST_FAIL; + } + } + } + atomic_store_explicit(&thread_ctx.stop_contender, true, + memory_order_release); + for (i = 0; !join_timed_out && i < nr_contenders_started; i++) { + err = join_thread(contenders[i], &join_deadline, &thread_err); + if (err == ETIMEDOUT) { + SCX_ERR("Timed out waiting for contender on CPU %d", + contender_ctxs[i].cpu); + join_timed_out = true; + status = SCX_TEST_FAIL; + } else if (err) { + SCX_ERR("Failed to join contender on CPU %d (%d)", + contender_ctxs[i].cpu, err); + status = SCX_TEST_FAIL; + } else { + nr_contenders_joined++; + if (thread_err) { + SCX_ERR("Contender on CPU %d failed (%d)", + contender_ctxs[i].cpu, thread_err); + status = SCX_TEST_FAIL; + } + } + } + + /* Restore the fair scheduler before waiting for any stranded thread. */ + if (join_timed_out) { + atomic_store_explicit(&thread_ctx.abort, true, + memory_order_release); + if (link) { + bpf_link__destroy(link); + link = NULL; + } + if (donor_started) + pthread_join(donor, NULL); + if (owner_started) + pthread_join(owner, NULL); + for (i = nr_contenders_joined; + i < nr_contenders_started; i++) + pthread_join(contenders[i], NULL); + } + + if (ioctl(thread_ctx.fd, ENQ_BLOCKED_IOCTL_GET_STATS, &stats)) { + SCX_ERR("Failed to read mutex statistics (%d)", errno); + status = SCX_TEST_FAIL; + } else { + result->stats = stats; + printf("\n[topology=%s SCX_OPS_ENQ_BLOCKED=%s]\n", + cross_cpu ? "cross-cpu" : "same-cpu", + enq_blocked ? "enabled" : "disabled"); + printf(" proxy_exec=%s\n", + proxy_enabled ? "enabled" : "disabled"); + printf(" donor_cpu=%d\n", thread_ctx.donor_cpu); + printf(" owner_cpu=%d\n", thread_ctx.owner_cpu); + printf(" nr_contenders=%zu\n", nr_contenders); + printf(" measured_trials=%d\n", NR_MEASURED_TRIALS); + printf(" owner_nice=%d\n", OWNER_NICE); + printf(" donor_nice=%d\n", DONOR_NICE); + printf(" contender_nice=%d\n", CONTENDER_NICE); + print_avg_time("mutex_hold", stats.hold_time_ns, stats.nr_holds); + print_avg_time("mutex_wait", stats.wait_time_ns, stats.nr_waits); + if (stats.nr_holds != NR_MEASURED_TRIALS || + stats.nr_waits != NR_MEASURED_TRIALS) { + SCX_ERR("Expected %d measured trials, got %llu holds and %llu waits", + NR_MEASURED_TRIALS, + (unsigned long long)stats.nr_holds, + (unsigned long long)stats.nr_waits); + status = SCX_TEST_FAIL; + } + } + + nr_blocked = skel->bss->nr_blocked_enqueues; + nr_blocked_donor_cpu = skel->bss->nr_blocked_enqueues_donor_cpu; + nr_blocked_owner_cpu = skel->bss->nr_blocked_enqueues_owner_cpu; + nr_blocked_other_cpu = skel->bss->nr_blocked_enqueues_other_cpu; + nr_blocked_wakeups = skel->bss->nr_blocked_wakeups; + result->nr_blocked_enqueues = nr_blocked; + result->nr_blocked_enqueues_donor_cpu = nr_blocked_donor_cpu; + result->nr_blocked_enqueues_owner_cpu = nr_blocked_owner_cpu; + result->nr_blocked_enqueues_other_cpu = nr_blocked_other_cpu; + result->nr_blocked_wakeups = nr_blocked_wakeups; + printf(" nr_blocked_enqueues=%llu\n", + (unsigned long long)nr_blocked); + printf(" nr_blocked_enqueues_donor_cpu=%llu\n", + (unsigned long long)nr_blocked_donor_cpu); + printf(" nr_blocked_enqueues_owner_cpu=%llu\n", + (unsigned long long)nr_blocked_owner_cpu); + printf(" nr_blocked_enqueues_other_cpu=%llu\n", + (unsigned long long)nr_blocked_other_cpu); + printf(" nr_blocked_wakeups=%llu\n", + (unsigned long long)nr_blocked_wakeups); + if (status == SCX_TEST_PASS) { + if (enq_blocked && proxy_enabled && !nr_blocked) { + SCX_ERR("ops.enqueue() did not receive the blocked donor"); + status = SCX_TEST_FAIL; + } else if ((!enq_blocked || !proxy_enabled) && nr_blocked) { + SCX_ERR("ops.enqueue() unexpectedly received %llu blocked donors", + (unsigned long long)nr_blocked); + status = SCX_TEST_FAIL; + } else if (nr_blocked_wakeups) { + SCX_ERR("Ordinary wakeups received %llu blocked enqueue flags", + (unsigned long long)nr_blocked_wakeups); + status = SCX_TEST_FAIL; + } else if (nr_blocked_other_cpu) { + SCX_ERR("Blocked donor had %llu enqueues on unexpected CPUs", + (unsigned long long)nr_blocked_other_cpu); + status = SCX_TEST_FAIL; + } + } + + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", + (unsigned long long)skel->data->uei.kind, + (long long)skel->data->uei.exit_code); + status = SCX_TEST_FAIL; + } + + if (link) + bpf_link__destroy(link); +out_fd: + close(thread_ctx.fd); +out_module: + unload_test_module(module_loaded); +out_skel: + enq_blocked__destroy(skel); +out_contenders: + free(contender_ctxs); + free(contenders); + return status; +} + +static enum scx_test_status run_topology(bool cross_cpu) +{ + struct run_result disabled = {}, enabled = {}; + enum scx_test_status status; + + status = run_one(false, cross_cpu, &disabled); + if (status != SCX_TEST_PASS) + return status; + + status = run_one(true, cross_cpu, &enabled); + if (status != SCX_TEST_PASS) + return status; + + printf("\n[topology=%s delta: enabled - disabled]\n", + cross_cpu ? "cross-cpu" : "same-cpu"); + print_avg_delta("mutex_hold", disabled.stats.hold_time_ns, + disabled.stats.nr_holds, enabled.stats.hold_time_ns, + enabled.stats.nr_holds); + print_avg_delta("mutex_wait", disabled.stats.wait_time_ns, + disabled.stats.nr_waits, enabled.stats.wait_time_ns, + enabled.stats.nr_waits); + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx) +{ + enum scx_test_status status; + + (void)ctx; + + status = run_topology(false); + if (status != SCX_TEST_PASS) + return status; + + status = run_topology(true); + if (status == SCX_TEST_SKIP) + return SCX_TEST_PASS; + + return status; +} + +struct scx_test enq_blocked = { + .name = "enq_blocked", + .description = "Verify proxy donor admission under CPU-wide contention", + .setup = setup, + .run = run, +}; + +REGISTER_SCX_TEST(&enq_blocked) diff --git a/tools/testing/selftests/sched_ext/enq_blocked.h b/tools/testing/selftests/sched_ext/enq_blocked.h new file mode 100644 index 000000000000..ef1eb97feebe --- /dev/null +++ b/tools/testing/selftests/sched_ext/enq_blocked.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES */ +#ifndef __ENQ_BLOCKED_H +#define __ENQ_BLOCKED_H + +#include +#include + +struct enq_blocked_stats { + __u64 hold_time_ns; + __u64 wait_time_ns; + __u64 nr_holds; + __u64 nr_waits; +}; + +#define ENQ_BLOCKED_IOCTL_OWNER _IO('s', 1) +#define ENQ_BLOCKED_IOCTL_DONOR _IO('s', 2) +#define ENQ_BLOCKED_IOCTL_RESET_STATS _IO('s', 3) +#define ENQ_BLOCKED_IOCTL_GET_STATS _IOR('s', 4, struct enq_blocked_stats) +#define ENQ_BLOCKED_IOCTL_PREP_ATTACH _IO('s', 5) +#define ENQ_BLOCKED_IOCTL_ATTACH_DONE _IO('s', 6) +#define ENQ_BLOCKED_IOCTL_DONOR_STATE _IO('s', 7) +#define ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED _IO('s', 8) + +#define ENQ_BLOCKED_DONOR_SLEEPING (1U << 0) +#define ENQ_BLOCKED_DONOR_ON_RQ (1U << 1) + +#endif /* __ENQ_BLOCKED_H */ diff --git a/tools/testing/selftests/sched_ext/test_modules/Makefile b/tools/testing/selftests/sched_ext/test_modules/Makefile new file mode 100644 index 000000000000..a0e9e9401ead --- /dev/null +++ b/tools/testing/selftests/sched_ext/test_modules/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + +TESTMODS_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +KDIR ?= $(if $(O),$(O),$(realpath ../../../../..)) + +obj-m += scx_enq_blocked_test.o + +all: + +$(Q)$(MAKE) -C $(KDIR) M=$(TESTMODS_DIR) modules + +clean: + +$(Q)$(MAKE) -C $(KDIR) M=$(TESTMODS_DIR) clean diff --git a/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c b/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c new file mode 100644 index 000000000000..908689ed5578 --- /dev/null +++ b/tools/testing/selftests/sched_ext/test_modules/scx_enq_blocked_test.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES + * + * Kernel mutex fixture for the sched_ext SCX_OPS_ENQ_BLOCKED selftest. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../enq_blocked.h" + +#define DONOR_WAIT_TIMEOUT msecs_to_jiffies(2000) +#define ATTACH_WAIT_TIMEOUT msecs_to_jiffies(10000) +#define MUTEX_HOLD_TIME msecs_to_jiffies(200) + +static DEFINE_MUTEX(test_mutex); +static DEFINE_SPINLOCK(donor_lock); +static struct task_struct *donor_task; +static atomic_t owner_ready = ATOMIC_INIT(0); +static atomic_t donor_started = ATOMIC_INIT(0); +static atomic_t attach_pending = ATOMIC_INIT(0); +static atomic_t attach_done = ATOMIC_INIT(0); +static atomic64_t hold_time_ns = ATOMIC64_INIT(0); +static atomic64_t wait_time_ns = ATOMIC64_INIT(0); +static atomic64_t nr_holds = ATOMIC64_INIT(0); +static atomic64_t nr_waits = ATOMIC64_INIT(0); + +static long run_owner(void) +{ + unsigned long timeout; + u64 start_ns; + long ret = 0; + + atomic_set(&donor_started, 0); + mutex_lock(&test_mutex); + start_ns = ktime_get_ns(); + atomic_set(&owner_ready, 1); + + timeout = jiffies + DONOR_WAIT_TIMEOUT; + while (!atomic_read(&donor_started)) { + if (time_after(jiffies, timeout)) { + ret = -ETIMEDOUT; + goto out; + } + cond_resched(); + } + if (atomic_xchg(&attach_pending, 0)) { + timeout = jiffies + ATTACH_WAIT_TIMEOUT; + while (!atomic_read(&attach_done)) { + if (time_after(jiffies, timeout)) { + ret = -ETIMEDOUT; + goto out; + } + cond_resched(); + } + } + + /* Keep yielding while the donor blocks on test_mutex. */ + timeout = jiffies + MUTEX_HOLD_TIME; + while (time_before(jiffies, timeout)) + cond_resched(); + +out: + atomic_set(&owner_ready, 0); + atomic64_add(ktime_get_ns() - start_ns, &hold_time_ns); + atomic64_inc(&nr_holds); + mutex_unlock(&test_mutex); + return ret; +} + +static long run_donor(void) +{ + unsigned long flags; + u64 start_ns; + + if (!atomic_read(&owner_ready)) + return -EAGAIN; + + get_task_struct(current); + spin_lock_irqsave(&donor_lock, flags); + WARN_ON_ONCE(donor_task); + donor_task = current; + spin_unlock_irqrestore(&donor_lock, flags); + + atomic_set(&donor_started, 1); + start_ns = ktime_get_ns(); + mutex_lock(&test_mutex); + + spin_lock_irqsave(&donor_lock, flags); + donor_task = NULL; + spin_unlock_irqrestore(&donor_lock, flags); + put_task_struct(current); + + atomic64_add(ktime_get_ns() - start_ns, &wait_time_ns); + atomic64_inc(&nr_waits); + mutex_unlock(&test_mutex); + return 0; +} + +static long get_donor_state(void) +{ + struct task_struct *task; + unsigned long flags; + long state = 0; + + spin_lock_irqsave(&donor_lock, flags); + task = donor_task; + if (task) + get_task_struct(task); + spin_unlock_irqrestore(&donor_lock, flags); + if (!task) + return -ENOENT; + + if (READ_ONCE(task->__state) != TASK_RUNNING) + state |= ENQ_BLOCKED_DONOR_SLEEPING; + if (READ_ONCE(task->on_rq)) + state |= ENQ_BLOCKED_DONOR_ON_RQ; + put_task_struct(task); + return state; +} + +static void reset_stats(void) +{ + atomic64_set(&hold_time_ns, 0); + atomic64_set(&wait_time_ns, 0); + atomic64_set(&nr_holds, 0); + atomic64_set(&nr_waits, 0); +} + +static long get_stats(unsigned long arg) +{ + struct enq_blocked_stats stats = { + .hold_time_ns = atomic64_read(&hold_time_ns), + .wait_time_ns = atomic64_read(&wait_time_ns), + .nr_holds = atomic64_read(&nr_holds), + .nr_waits = atomic64_read(&nr_waits), + }; + + return copy_to_user((void __user *)arg, &stats, sizeof(stats)) ? + -EFAULT : 0; +} + +static long enq_blocked_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case ENQ_BLOCKED_IOCTL_OWNER: + return run_owner(); + case ENQ_BLOCKED_IOCTL_DONOR: + return run_donor(); + case ENQ_BLOCKED_IOCTL_RESET_STATS: + reset_stats(); + return 0; + case ENQ_BLOCKED_IOCTL_GET_STATS: + return get_stats(arg); + case ENQ_BLOCKED_IOCTL_PREP_ATTACH: + atomic_set(&attach_done, 0); + atomic_set(&attach_pending, 1); + return 0; + case ENQ_BLOCKED_IOCTL_ATTACH_DONE: + atomic_set(&attach_done, 1); + return 0; + case ENQ_BLOCKED_IOCTL_DONOR_STATE: + return get_donor_state(); + case ENQ_BLOCKED_IOCTL_PROXY_SUPPORTED: + return IS_ENABLED(CONFIG_SCHED_PROXY_EXEC); + default: + return -EINVAL; + } +} + +static const struct file_operations enq_blocked_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = enq_blocked_ioctl, +}; + +static struct miscdevice enq_blocked_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "scx_enq_blocked", + .fops = &enq_blocked_fops, + .mode = 0600, +}; + +module_misc_device(enq_blocked_device); +MODULE_AUTHOR("Andrea Righi "); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("sched_ext blocked donor test module"); From b58f3d2d5d881d17338936c9b5e589bc258269f3 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:54 +0200 Subject: [PATCH 40/69] sched_ext: scx_qmap: Add proxy execution support Add a -X option to opt scx_qmap into queueing mutex-blocked tasks for proxy execution. Without the option, SCX_OPS_ENQ_BLOCKED remains clear and mutex waiters block normally. With -X, blocked donors are passed to qmap_enqueue() with SCX_ENQ_BLOCKED. When scx_qmap receives a blocked donor, select a cid allowed by the task and held by qmap, preferring its current cid. Dispatch the donor to that cid's local DSQ with a fresh slice and SCX_ENQ_PREEMPT, adding SCX_ENQ_IMMED when the cid is time-shared. This places the donor at the head of an eligible DSQ and requests an immediate reschedule, allowing the core proxy-exec path to run the mutex owner using the donor's scheduling context as soon as the donor is selected. Partition ownership can change concurrently with enqueue, so the self-cid intersection can disappear between the initial check and cid selection. Validate the selected cid before indexing per-cid state or constructing a local DSQ id, and fall back to rescue placement if the intersection has disappeared. The blocked policy is intentionally unfair and can strongly prioritize tasks using contended mutexes, but scx_qmap is a demo scheduler and such aggressive behavior makes proxy-exec support easy to observe. Count all blocked dispatch attempts in nr_enq_blocked and report their per-interval delta. Acked-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- tools/sched_ext/scx_qmap.bpf.c | 79 +++++++++++++++++++++++++++++++--- tools/sched_ext/scx_qmap.c | 13 ++++-- tools/sched_ext/scx_qmap.h | 1 + 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 2f3653199842..81575294027e 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -369,6 +369,22 @@ static u64 needs_immed(s32 cid) return qa.cid_shared[cid] ? SCX_ENQ_IMMED : 0; } +static void dispatch_to_rescue(struct task_struct *p, task_ctx_t *taskc, + u64 enq_flags) +{ + u32 cid = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + + if (cid >= scx_bpf_nr_cids()) { + scx_bpf_error("task %d has no allowed cid", p->pid); + return; + } + + taskc->force_local = false; + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, + enq_flags | SCX_ENQ_RESCUE); +} + /* first cid this node does NOT hold for fault injection, -1 if none */ static s32 first_unavail_cid(void) { @@ -442,6 +458,9 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) */ taskc->core_sched_seq = qa.core_sched_tail_seqs[idx]++; + if (enq_flags & SCX_ENQ_BLOCKED) + __sync_fetch_and_add(&qa.nr_enq_blocked, 1); + /* * A task of ours that can run on none of our self cids - the parent * didn't grant them or we delegated them to children - would starve in @@ -451,18 +470,64 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags) * If we hold ENQ on that cid it runs. Otherwise the kernel diverts the * task to its rescue path. IMMED would turn the insert into a legal * placement on a time-shared cid and the kernel would bounce it back - * here instead of rescuing it. + * here instead of rescuing it. Do this before the blocked-donor fast + * paths, which also require an eligible self cid to make progress. */ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { - s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + dispatch_to_rescue(p, taskc, enq_flags); + return; + } - if (c >= 0 && c < scx_bpf_nr_cids()) { - taskc->force_local = false; - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns, - enq_flags | SCX_ENQ_RESCUE); + /* + * SCX_OPS_ALWAYS_ENQ_IMMED makes the local insertion below implicitly + * carry SCX_ENQ_IMMED. If the CPU can't run the blocked donor immediately, + * the core returns it through ops.enqueue() with SCX_ENQ_REENQ. Inserting + * it into the same local DSQ would repeat the IMMED handback until the + * scheduler is ejected. Move reenqueued blocked donors to the shared DSQ, + * which doesn't carry SCX_ENQ_IMMED, so another CPU can consume them. + */ + if ((enq_flags & (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) == + (SCX_ENQ_BLOCKED | SCX_ENQ_REENQ)) { + taskc->force_local = false; + scx_bpf_dsq_insert(p, SHARED_DSQ, 0, enq_flags); + cid = cmask_next_and2_set_wrap(&taskc->cpus_allowed, + &qa.idle_cids.mask, + &qa.self_cids.mask, 0); + if (cid < scx_bpf_nr_cids()) + scx_bpf_kick_cid(cid, SCX_KICK_IDLE); + return; + } + + /* + * Insert a blocked mutex donor at the head of an eligible local DSQ with + * a fresh slice and %SCX_ENQ_PREEMPT, requesting an immediate reschedule. + * The test above guarantees that cpus_allowed intersects self_cids, but + * the donor's current cid may have been delegated to a child. Search the + * intersection starting at the current cid, preserving it when qmap still + * holds it and wrapping to another eligible self cid otherwise. + * + * A self cid may be held exclusively with SCX_CAP_ENQ or time-shared with + * only SCX_CAP_ENQ_IMMED. Add needs_immed() so either kind can accept the + * local insertion instead of rejecting and reenqueuing the donor for a + * capability miss. Once selected, the core proxy-exec path can run the + * mutex owner using the donor's scheduling context. + * + * This policy is intentionally unfair and can strongly prioritize tasks + * using contended mutexes; scx_qmap is a demonstration scheduler and + * this behavior makes proxy-exec support easy to observe. + */ + if (enq_flags & SCX_ENQ_BLOCKED) { + cid = cmask_next_and_set_wrap(&taskc->cpus_allowed, + &qa.self_cids.mask, + scx_bpf_task_cid(p)); + if (cid >= scx_bpf_nr_cids()) { + /* self_cids may have changed since the intersection test */ + dispatch_to_rescue(p, taskc, enq_flags); return; } + scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | cid, slice_ns, + enq_flags | needs_immed(cid) | SCX_ENQ_PREEMPT); + return; } /* diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index 5bb5f687e579..d5226e071657 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -46,7 +46,7 @@ const char help_fmt[] = "See the top-of-file comment in .bpf.c for the design.\n" "\n" "Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n" -" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I]\n" +" [-N COUNT] [-P] [-M] [-H] [-c CG_PATH] [-d PID] [-D LEN] [-S] [-p] [-I] [-X]\n" " [-F COUNT] [-i SEC] [-R MS] [-J MODE] [-v]\n" "\n" " -s SLICE_US Override slice duration\n" @@ -65,6 +65,7 @@ const char help_fmt[] = " -S Suppress qmap-specific debug dump\n" " -p Switch only tasks on SCHED_EXT policy instead of all\n" " -I Turn on SCX_OPS_ALWAYS_ENQ_IMMED\n" +" -X Turn on SCX_OPS_ENQ_BLOCKED\n" " -F COUNT IMMED stress: force every COUNT'th enqueue to a busy local DSQ (use with -I)\n" " -C MODE cid-override test (shuffle|bad-dup|bad-range|bad-mono)\n" " -i SEC Stats interval, seconds (default 5)\n" @@ -107,6 +108,7 @@ struct hier_prev { u64 nr_dsps[MAX_SUB_SCHEDS]; u64 nr_reenq_cap; u64 nr_reenq_immed; + u64 nr_enq_blocked; u64 nr_inject_attempts; u64 nr_rescue_dsp; }; @@ -190,14 +192,16 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg } format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges)); - printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu rescue=+%llu\n", + printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu blocked=+%llu inj=+%llu rescue=+%llu\n", (unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr, (unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap), (unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed), + (unsigned long long)(qa->nr_enq_blocked - prev->nr_enq_blocked), (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts), (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp)); prev->nr_reenq_cap = qa->nr_reenq_cap; prev->nr_reenq_immed = qa->nr_reenq_immed; + prev->nr_enq_blocked = qa->nr_enq_blocked; prev->nr_inject_attempts = qa->nr_inject_attempts; prev->nr_rescue_dsp = qa->nr_rescue_dsp; @@ -263,7 +267,7 @@ int main(int argc, char **argv) skel->rodata->max_tasks = 16384; while ((opt = getopt(argc, argv, - "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:B:q:vh")) != -1) { + "s:e:t:T:l:b:N:PMHc:d:D:SpIXF:C:i:R:J:B:q:vh")) != -1) { switch (opt) { case 's': skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000; @@ -324,6 +328,9 @@ int main(int argc, char **argv) case 'I': skel->struct_ops.qmap_ops->flags |= SCX_OPS_ALWAYS_ENQ_IMMED; break; + case 'X': + skel->struct_ops.qmap_ops->flags |= SCX_OPS_ENQ_BLOCKED; + break; case 'F': skel->rodata->immed_stress_nth = strtoul(optarg, NULL, 0); break; diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index e95fffcf7b23..089c5176c4ec 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -181,6 +181,7 @@ struct qmap_arena { /* bpf -> userspace: stats */ u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */ u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */ + u64 nr_enq_blocked; /* SCX_ENQ_BLOCKED dispatches */ u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */ u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */ u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */ From c6fe97c34a1a928bd1f6bbf47e6fe7130c501338 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 22 Sep 2026 18:51:55 +0200 Subject: [PATCH 41/69] sched: Allow enabling proxy exec with sched_ext Now that sched_ext can handle proxy donors and BPF schedulers can opt in to blocked-donor enqueueing with SCX_OPS_ENQ_BLOCKED, remove the !SCHED_CLASS_EXT dependency from SCHED_PROXY_EXEC and allow both options to be enabled together. Acked-by: John Stultz Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- init/Kconfig | 2 -- 1 file changed, 2 deletions(-) diff --git a/init/Kconfig b/init/Kconfig index 8583d9f06c52..f3b3bbd14dc4 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -938,8 +938,6 @@ config SCHED_PROXY_EXEC bool "Proxy Execution" # Avoid some build failures w/ PREEMPT_RT until it can be fixed depends on !PREEMPT_RT - # Need to investigate how to inform sched_ext of split contexts - depends on !SCHED_CLASS_EXT # Not particularly useful until we get to multi-rq proxying depends on EXPERT help From f2f095c87603b8dfb6752e130d902970371c3852 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 29 Sep 2026 07:32:26 -1000 Subject: [PATCH 42/69] sched_ext: Work around pahole 1.32 dropping scx_bpf_task_set_lazy_resched() from BTF With clang and pahole 1.32, x86-64 vmlinux BTF lacks scx_bpf_task_set_lazy_resched() and sched_ext fails to initialize: sched_ext: Failed to register kfunc sets (-22) clang describes the lazy argument only after the prologue moves it into a callee-saved register, which pahole 1.32 rejects as not being the argument register. Released pahole is affected whatever the eventual fix. Keep the argument on the stack with barrier_data() so that its location names no register. OPTIMIZER_HIDE_VAR() would fix x86-64 but break arm64. Fixes: f8e5a4e3f3be ("sched_ext: Add lazy preemption support") Link: https://lore.kernel.org/r/20260925213619.2187751-1-yonghong.song@linux.dev Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index e48aac45afea..405d0d1038f8 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -10136,6 +10136,11 @@ __bpf_kfunc bool scx_bpf_task_set_lazy_resched(struct task_struct *p, bool lazy, { struct scx_sched *sch; + /* + * clang's register locations for @lazy make pahole 1.32 drop this kfunc + * from BTF. Keep @lazy in memory so its location names no register. + */ + barrier_data(&lazy); guard(rcu)(); sch = scx_prog_sched(aux); if (unlikely(!sch || !scx_task_on_sched(sch, p))) From e0bb535017713171b7c5c595bd08c2e8527de863 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 29 Sep 2026 13:20:29 -1000 Subject: [PATCH 43/69] sched_ext: Sync common and compat headers from the scx repo Sync cid.bpf.h, common.bpf.h, compat.bpf.h and compat.h with the scx repo at c9b90b3ad3d0 ("scheds/include: Sync with kernel sched_ext/for-7.4 (c6fe97c34a1a)"), which accumulated the following since the last sync: - cid.bpf.h grew the cmask helpers the cid-form schedulers use: fill, empty and end tests, unchecked and word-level access, distribute picks over an arena rotor indexed by cid, range set and clear, and three-operand and, or and andnot that report whether the result is empty. Its word loops walk with bpf_arena_for() and its bit helpers take the fetching atomics when the loader's probe finds the JIT lowers them. - The prolog probe's default now makes is_migration_disabled() over-report, which errs toward local-only dispatch instead of crashing, and the cid-form open macros repoint the probe at bpf_scx_reg_cid(), which cid-form schedulers register through. - scx_bpf_kick_cid() and scx_bpf_cidperf_set() changed their return types without being renamed, so compat.bpf.h declares both flavors of each and lets the kernel pick, the way scx_bpf_dsq_insert() is handled. - A macro supplies scx_bpf_cid_topo()'s size argument from the output buffer, TRAILING_OVERLAP() embeds a struct that ends in a flexible array together with its storage, bpf_arena_for() walks a range in a form the verifier converges on, and features.bpf.h carries the kernel feature bits the loader probes before load. const-defs.h holds the per-architecture cacheline size that cid.bpf.h aligns its rotor slots to. scx_qmap already used the cmask helpers, and the three-operand forms of cmask_and() and cmask_andnot() had not reached its copy of cid.bpf.h, so tools/sched_ext no longer built against scx's headers. Its five calls take the three-operand form now, with the copy-then-operate pairs folded into one call each. Signed-off-by: Tejun Heo --- tools/sched_ext/include/scx/cid.bpf.h | 697 ++++++++++++++++++--- tools/sched_ext/include/scx/common.bpf.h | 90 ++- tools/sched_ext/include/scx/compat.bpf.h | 63 ++ tools/sched_ext/include/scx/compat.h | 15 +- tools/sched_ext/include/scx/const-defs.h | 20 + tools/sched_ext/include/scx/features.bpf.h | 29 + tools/sched_ext/scx_qmap.bpf.c | 14 +- 7 files changed, 812 insertions(+), 116 deletions(-) create mode 100644 tools/sched_ext/include/scx/const-defs.h create mode 100644 tools/sched_ext/include/scx/features.bpf.h diff --git a/tools/sched_ext/include/scx/cid.bpf.h b/tools/sched_ext/include/scx/cid.bpf.h index 69fb4e97bc77..d1e6ad1432d3 100644 --- a/tools/sched_ext/include/scx/cid.bpf.h +++ b/tools/sched_ext/include/scx/cid.bpf.h @@ -3,8 +3,8 @@ * BPF-side helpers for cids and cmasks. See kernel/sched/ext/cid.h for the * authoritative layout and semantics. The BPF-side helpers use the cmask_* * naming (no scx_ prefix); cmask is the SCX bitmap type so the prefix is - * redundant in BPF code. Atomics use __sync_val_compare_and_swap and every - * helper is inline (no .c counterpart). + * redundant in BPF code. Every helper is inline except the binary operations, + * which are weak global functions defined here, so there is no .c counterpart. * * Included by scx/common.bpf.h; don't include directly. * @@ -16,6 +16,11 @@ #include "bpf_arena_common.bpf.h" +/* libbpf's bpf_helpers.h defines it from 1.4 on */ +#ifndef __arg_arena +#define __arg_arena __attribute((btf_decl_tag("arg:arena"))) +#endif + #ifndef BIT_U64 #define BIT_U64(nr) (1ULL << (nr)) #endif @@ -76,7 +81,7 @@ static __always_inline void __cmask_init(struct scx_cmask __arena *m, u32 base, m->nr_cids = nr_cids; m->alloc_words = alloc_words; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= alloc_words) break; m->bits[i] = 0; @@ -122,24 +127,115 @@ static __always_inline void cmask_reframe(struct scx_cmask __arena *m, u32 base, m->nr_cids = nr_cids; } +/** + * cmask_nr_words - Words of bits[] that @m's active range spans + * @m: cmask to measure + * + * @m->base need not be word aligned: bits[0] covers the whole word @m->base + * falls in, so the count is taken from that word, not from @m->base. + */ +static __always_inline u32 cmask_nr_words(const struct scx_cmask __arena *m) +{ + u32 wbase = m->base / 64; + + return m->nr_cids ? (m->base + m->nr_cids - 1) / 64 - wbase + 1 : 0; +} + +/** + * cmask_word - Read word @k of @m + * @m: cmask to read + * @k: word index, counted from the word @m->base falls in + * + * A word past the active range reads as 0, so a scan that looks one word + * ahead (at an SMT sibling that falls in the next word, say) needs no bound + * of its own. + */ +static __always_inline u64 cmask_word(const struct scx_cmask __arena *m, u32 k) +{ + if (k >= cmask_nr_words(m)) + return 0; + return m->bits[k]; +} + +/** + * cmask_range_word - Bits of word @k of @m that fall in [@start, @start + @nr) + * @m: cmask the word belongs to + * @k: word index, counted from the word @m->base falls in + * @start: first cid of the range + * @nr: number of cids in the range + * + * Scoping a scan to a domain that is contiguous in cid space, an LLC or a + * node, is then one AND per word, rather than a test of the ends of the + * range at every bit or a mask kept per domain. + */ +static __always_inline u64 cmask_range_word(const struct scx_cmask __arena *m, + u32 k, u32 start, u32 nr) +{ + u64 wlo = (u64)(m->base / 64 + k) * 64, whi = wlo + 64; + u64 lo = start, hi = (u64)start + nr; + + if (lo < wlo) + lo = wlo; + if (hi > whi) + hi = whi; + if (lo >= hi) + return 0; + + return GENMASK_U64(hi - wlo - 1, lo - wlo); +} + +/** + * __cmask_test - Test a cid without checking it against the active range + * @cid: cid to test + * @m: cmask to test + * + * The caller must already know that @cid lies within + * [@m->base, @m->base + @m->nr_cids), or the read runs off @m->bits. + * + * For hot scans that have already bounded @cid, where the per-call range check + * is repeated work on every candidate. + */ +static __always_inline bool __cmask_test(u32 cid, const struct scx_cmask __arena *m) +{ + return *__cmask_word(cid, m) & BIT_U64(cid & 63); +} + static __always_inline bool cmask_test(u32 cid, const struct scx_cmask __arena *m) { if (!__cmask_contains(cid, m)) return false; - return *__cmask_word(cid, m) & BIT_U64(cid & 63); + return __cmask_test(cid, m); } /* - * x86 BPF JIT rejects BPF_OR | BPF_FETCH and BPF_AND | BPF_FETCH on arena - * pointers (see bpf_jit_supports_insn() in arch/x86/net/bpf_jit_comp.c). Only - * BPF_CMPXCHG / BPF_XCHG / BPF_ADD with FETCH are allowed. Implement - * test_and_{set,clear} and the atomic set/clear via a cmpxchg loop. + * The bit helpers use the fetching bitwise builtins, which order like the + * cmpxchg they replace, and not every JIT accepts those on arena pointers. The + * loader sets SCX_LIB_FEAT_ARENA_FETCH_BITOPS when a probe with the fetching + * form loads, and the helpers run a cmpxchg loop otherwise. The fetched value + * is consumed through barrier_var() because clang 19 lowers a builtin whose + * result is unused to the non-fetching form, which is relaxed on arm64. * * CMASK_CAS_TRIES is sized so exhausting it means seconds of real spinning - * on one word - past any plausible contention. Abort hard. + * on one word, past any plausible contention. Abort hard. */ #define CMASK_CAS_TRIES (1U << 23) +static __always_inline u64 __cmask_fetch_or(u64 __arena *w, u64 mask) +{ + u64 old = __atomic_fetch_or(w, mask, __ATOMIC_ACQ_REL); + + barrier_var(old); + return old; +} + +static __always_inline u64 __cmask_fetch_andnot(u64 __arena *w, u64 mask) +{ + u64 old = __atomic_fetch_and(w, ~mask, __ATOMIC_ACQ_REL); + + barrier_var(old); + return old; +} + static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) { u64 __arena *w; @@ -150,7 +246,11 @@ static __always_inline void cmask_set(u32 cid, struct scx_cmask __arena *m) return; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_or(w, bit); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (old & bit) return; @@ -171,7 +271,11 @@ static __always_inline void cmask_clear(u32 cid, struct scx_cmask __arena *m) return; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_andnot(w, bit); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (!(old & bit)) return; @@ -192,7 +296,9 @@ static __always_inline bool cmask_test_and_set(u32 cid, struct scx_cmask __arena return false; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) + return __cmask_fetch_or(w, bit) & bit; + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (old & bit) return true; @@ -214,7 +320,9 @@ static __always_inline bool cmask_test_and_clear(u32 cid, struct scx_cmask __are return false; w = __cmask_word(cid, m); bit = BIT_U64(cid & 63); - bpf_for(i, 0, CMASK_CAS_TRIES) { + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) + return __cmask_fetch_andnot(w, bit) & bit; + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { old = *w; if (!(old & bit)) return false; @@ -268,17 +376,242 @@ static __always_inline bool __cmask_test_and_clear(u32 cid, struct scx_cmask __a return prev; } +/* atomically or @mask into *@w */ +static __always_inline void __cmask_word_or(u64 __arena *w, u64 mask) +{ + u64 old, new; + u32 i; + + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_or(w, mask); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { + old = *w; + if ((old & mask) == mask) + return; + new = old | mask; + if (__sync_val_compare_and_swap(w, old, new) == old) + return; + } + scx_bpf_error("__cmask_word_or CAS exhausted"); +} + +/* atomically clear @mask bits in *@w */ +static __always_inline void __cmask_word_andnot(u64 __arena *w, u64 mask) +{ + u64 old, new; + u32 i; + + if (scx_lib_has(SCX_LIB_FEAT_ARENA_FETCH_BITOPS)) { + __cmask_fetch_andnot(w, mask); + return; + } + bpf_arena_for(i, 0, CMASK_CAS_TRIES) { + old = *w; + if (!(old & mask)) + return; + new = old & ~mask; + if (__sync_val_compare_and_swap(w, old, new) == old) + return; + } + scx_bpf_error("__cmask_word_andnot CAS exhausted"); +} + +/** + * cmask_full_range - Test whether every cid in [@start, @start + @nr) is set + * @m: cmask to test + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. True when no clamped bit is + * clear, including when the clamped range is empty. + */ +static __always_inline bool cmask_full_range(const struct scx_cmask __arena *m, u32 start, + u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return true; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + if ((m->bits[wi] & mask) != mask) + return false; + } + return true; +} + +/** + * cmask_set_range - Set every cid in [@start, @start + @nr) + * @m: cmask to modify + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. Words are updated atomically + * so concurrent updates of other bits sharing a word are never lost. The range + * as a whole does not transition atomically. + */ +static __always_inline void cmask_set_range(struct scx_cmask __arena *m, u32 start, u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + __cmask_word_or(&m->bits[wi], mask); + } +} + +/** + * cmask_clear_range - Clear every cid in [@start, @start + @nr) + * @m: cmask to modify + * @start: first cid of the range + * @nr: number of cids in the range + * + * The range is clamped to @m's active range first. Words are updated atomically + * so concurrent updates of other bits sharing a word are never lost. The range + * as a whole does not transition atomically. + */ +static __always_inline void cmask_clear_range(struct scx_cmask __arena *m, u32 start, u32 nr) +{ + u64 end = (u64)start + nr; + u32 wbase = m->base / 64; + u32 first_wi, last_wi, first_bit, last_bit, last, i; + + if (start < m->base) + start = m->base; + if (end > m->base + m->nr_cids) + end = m->base + m->nr_cids; + if (start >= end) + return; + last = end - 1; + + first_wi = start / 64 - wbase; + last_wi = last / 64 - wbase; + first_bit = start & 63; + last_bit = last & 63; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + u32 wi = first_wi + i; + u64 mask = ~0ULL; + + if (wi > last_wi) + break; + if (wi == first_wi) + mask &= GENMASK_U64(63, first_bit); + if (wi == last_wi) + mask &= GENMASK_U64(last_bit, 0); + __cmask_word_andnot(&m->bits[wi], mask); + } +} + static __always_inline void cmask_zero(struct scx_cmask __arena *m) { u32 nr_words = CMASK_NR_WORDS(m->nr_cids), i; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; m->bits[i] = 0; } } +/** + * cmask_fill - Set every cid in @m's active range + * @m: cmask to fill + * + * Counterpart to cmask_zero(). Storage past the active range is left as is, + * matching the kernel-side scx_cmask_fill(). + */ +static __always_inline void cmask_fill(struct scx_cmask __arena *m) +{ + u32 nr_words, head_bits, tail_bits, i; + + if (!m->nr_cids) + return; + nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + if (i >= nr_words) + break; + m->bits[i] = ~0LLU; + } + + /* clear word-0 bits below base */ + head_bits = m->base & 63; + if (head_bits) + m->bits[0] &= ~((1LLU << head_bits) - 1); + + /* clear last-word bits at or past base + nr_cids */ + tail_bits = (m->base + m->nr_cids) & 63; + if (tail_bits) + m->bits[nr_words - 1] &= (1LLU << tail_bits) - 1; +} + +/** + * cmask_empty - Test whether no cid is set in @m's active range + * @m: cmask to test + * + * Scans the words the active range spans, whose bits outside the range every + * mutator keeps zero. + */ +static __always_inline bool cmask_empty(const struct scx_cmask __arena *m) +{ + u32 nr_words = cmask_nr_words(m), i; + + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { + if (i >= nr_words) + break; + if (m->bits[i]) + return false; + } + return true; +} + /* * BPF_-prefixed to avoid colliding with the kernel's anonymous CMASK_OP_* * enum in ext/cid.c, which is exported via BTF and reachable through @@ -291,92 +624,136 @@ enum { BPF_CMASK_OP_ANDNOT, }; -static __always_inline void cmask_op_word(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src, - u32 di, u32 si, u64 mask, int op) +/* range bits of word @k of @m, which spans @nr words: 0 past the span */ +static __always_inline u64 __cmask_word_range(const struct scx_cmask __arena *m, u32 k, + u32 nr) { - u64 dv = dst->bits[di]; - u64 sv = src->bits[si]; - u64 rv; + u64 bits = ~0ULL; - if (op == BPF_CMASK_OP_AND) - rv = dv & sv; - else if (op == BPF_CMASK_OP_OR) - rv = dv | sv; - else if (op == BPF_CMASK_OP_ANDNOT) - rv = dv & ~sv; - else - rv = sv; - - dst->bits[di] = (dv & ~mask) | (rv & mask); -} - -static __always_inline void cmask_op(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src, int op) -{ - u32 d_end = dst->base + dst->nr_cids; - u32 s_end = src->base + src->nr_cids; - u32 lo = dst->base > src->base ? dst->base : src->base; - u32 hi = d_end < s_end ? d_end : s_end; - u32 d_base = dst->base / 64; - u32 s_base = src->base / 64; - u32 lo_word, hi_word, i; - u64 head_mask, tail_mask; - - if (lo >= hi) - return; - - lo_word = lo / 64; - hi_word = (hi - 1) / 64; - head_mask = GENMASK_U64(63, lo & 63); - tail_mask = GENMASK_U64((hi - 1) & 63, 0); - - bpf_for(i, 0, CMASK_MAX_WORDS) { - u32 w = lo_word + i; - u64 m; - - if (w > hi_word) - break; - - m = GENMASK_U64(63, 0); - if (w == lo_word) - m &= head_mask; - if (w == hi_word) - m &= tail_mask; - - cmask_op_word(dst, src, w - d_base, w - s_base, m, op); - } + if (k >= nr) + return 0; + if (k == 0) + bits &= GENMASK_U64(63, m->base & 63); + if (k == nr - 1) + bits &= GENMASK_U64((m->base + m->nr_cids - 1) & 63, 0); + return bits; } /* - * cmask_and/or/copy only modify @dst bits that lie in the intersection of - * [@dst->base, @dst->base + @dst->nr_cids) and [@src->base, - * @src->base + @src->nr_cids). Bits in @dst outside that window - * keep their prior values - in particular, cmask_copy() does NOT zero @dst - * bits that lie outside @src's range. + * The binary operations combine cmasks of any active ranges. A cmask holds bits + * only for its own range and reads as the operation's identity outside it, all + * ones for AND and all zeros for OR, so an operand alters only the bits it + * covers: ANDing a shard's mask into a global one edits that shard's window and + * leaves the rest of the global mask alone. ANDNOT is AND with @src2 inverted + * over @src2's range, so where only @src2 has range the result is ~@src2. + * + * @dst is written on its range intersected with the union of the sources' + * ranges and untouched elsewhere, and its padding bits outside its own range + * stay zero, the invariant every cmask mutator keeps. @dst may alias either + * source, and the two sources may be one mask. Returns whether @dst has a bit + * set afterwards, over all of @dst's range. + * + * The operations are global functions, verified once per program. Inlined, + * every data-dependent branch in this loop would multiply the states the + * verifier explores per iteration at every call site. */ -static __always_inline void cmask_and(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +static __always_inline bool __cmask_op(struct scx_cmask __arena *dst, + const struct scx_cmask __arena *src1, + const struct scx_cmask __arena *src2, int op) { - cmask_op(dst, src, BPF_CMASK_OP_AND); + u32 d_first = dst->base / 64, d_nr = cmask_nr_words(dst); + u32 s1_first = src1->base / 64, s1_nr = cmask_nr_words(src1); + u32 s2_first = src2->base / 64, s2_nr = cmask_nr_words(src2); + u64 any = bpf_arena_loop_zero; + u32 i; + + bpf_arena_for(i, 0, d_nr) { + u32 k1 = d_first + i - s1_first, k2 = d_first + i - s2_first; + u64 m1 = __cmask_word_range(src1, k1, s1_nr); + u64 m2 = op == BPF_CMASK_OP_COPY ? m1 : __cmask_word_range(src2, k2, s2_nr); + u64 um = __cmask_word_range(dst, i, d_nr) & (m1 | m2); + u64 dv = dst->bits[i], v1, v2, rv; + + if (um) { + /* a word in range holds zeros outside the range */ + v1 = m1 ? src1->bits[k1] : 0; + v2 = m2 ? src2->bits[k2] : 0; + if (op == BPF_CMASK_OP_AND) + rv = (v1 | ~m1) & (v2 | ~m2); + else if (op == BPF_CMASK_OP_OR) + rv = v1 | v2; + else if (op == BPF_CMASK_OP_ANDNOT) + rv = (v1 | ~m1) & ~v2; + else + rv = v1; + dv = (dv & ~um) | (rv & um); + dst->bits[i] = dv; + } + any |= dv; + } + return any != 0; } -static __always_inline void cmask_or(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_and - Store @src1 AND @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: first operand + * @src2: second operand + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. + */ +__weak bool cmask_and(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_OR); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_AND); } -static __always_inline void cmask_copy(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_or - Store @src1 OR @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: first operand + * @src2: second operand + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. + */ +__weak bool cmask_or(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_COPY); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_OR); } -static __always_inline void cmask_andnot(struct scx_cmask __arena *dst, - const struct scx_cmask __arena *src) +/** + * cmask_andnot - Store @src1 AND NOT @src2 in @dst + * @dst: destination, may be @src1 or @src2 + * @src1: operand to remove bits from + * @src2: bits to remove, inverted over its own range + * + * See __cmask_op() for how the ranges combine. Return whether @dst has a bit + * set afterwards. + */ +__weak bool cmask_andnot(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src1, + const struct scx_cmask __arena __arg_arena *src2) { - cmask_op(dst, src, BPF_CMASK_OP_ANDNOT); + return __cmask_op(dst, src1, src2, BPF_CMASK_OP_ANDNOT); +} + +/** + * cmask_copy - Copy @src into @dst + * @dst: destination + * @src: source + * + * The single-source case of __cmask_op(): @dst is written over @src's range and + * untouched elsewhere. + */ +__weak void cmask_copy(struct scx_cmask __arena __arg_arena *dst, + const struct scx_cmask __arena __arg_arena *src) +{ + __cmask_op(dst, src, src, BPF_CMASK_OP_COPY); } /* @@ -395,7 +772,7 @@ static __always_inline bool cmask_equal(const struct scx_cmask __arena *a, return true; nr_words = (a->base + a->nr_cids - 1) / 64 - a->base / 64 + 1; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; if (a->bits[i] != b->bits[i]) @@ -428,7 +805,7 @@ static __always_inline u32 cmask_next_set(const struct scx_cmask __arena *m, u32 start_wi = cid / 64 - base; start_bit = cid & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 wi = start_wi + i; u64 word; u32 found; @@ -457,7 +834,7 @@ static __always_inline u32 cmask_first_set(const struct scx_cmask __arena *m) #define cmask_for_each(cid, m) \ for ((cid) = cmask_first_set(m); \ - (cid) < (m)->base + (m)->nr_cids; \ + (cid) < (m)->base + (m)->nr_cids && can_loop; \ (cid) = cmask_next_set((m), (cid) + 1)) /* @@ -495,7 +872,7 @@ static __always_inline bool cmask_subset(const struct scx_cmask __arena *a, lo_word = lo / 64; hi_word = (hi - 1) / 64; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 w = lo_word + i; if (w > hi_word) @@ -514,13 +891,14 @@ static __always_inline bool cmask_subset(const struct scx_cmask __arena *a, static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m) { u32 nr_words, i; - u32 count = 0; + /* callers compare the sum, see bpf_arena_loop_zero */ + u32 count = bpf_arena_loop_zero; if (!m->nr_cids) return 0; nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { if (i >= nr_words) break; count += __builtin_popcountll(m->bits[i]); @@ -528,10 +906,7 @@ static __always_inline u32 cmask_weight(const struct scx_cmask __arena *m) return count; } -/* - * True if @a and @b share any set bit. Walk only the intersection of their - * ranges, matching the semantics of cmask_and(). - */ +/* true if @a and @b share any set bit, over the intersection of their ranges */ static __always_inline bool cmask_intersects(const struct scx_cmask __arena *a, const struct scx_cmask __arena *b) { @@ -552,7 +927,7 @@ static __always_inline bool cmask_intersects(const struct scx_cmask __arena *a, head_mask = GENMASK_U64(63, lo & 63); tail_mask = GENMASK_U64((hi - 1) & 63, 0); - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 w = lo_word + i; u64 mask, av, bv; @@ -603,7 +978,7 @@ static __always_inline u32 cmask_next_and_set(const struct scx_cmask __arena *a, start_wi = start / 64; start_bit = start & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 abs_wi = start_wi + i; u64 word; u32 found; @@ -646,6 +1021,17 @@ static __always_inline u32 cmask_next_set_wrap(const struct scx_cmask __arena *m return found < start ? found : end; } +/** + * cmask_end - One past the last cid of @m's active range + * @m: cmask of interest + * + * The scan helpers return this value when no set cid is found. + */ +static __always_inline u32 cmask_end(const struct scx_cmask __arena *m) +{ + return m->base + m->nr_cids; +} + /* * Find the next cid set in both @a and @b at or after @start, wrapping to * @a->base if none found in the forward half. Return a->base + a->nr_cids @@ -668,6 +1054,121 @@ static __always_inline u32 cmask_next_and_set_wrap(const struct scx_cmask __aren return found < start ? found : a_end; } +/* + * The distribute helpers rotate through one slot per cid, a cacheline each so + * picks on one CPU do not bounce the line of another. A missing allocation is a + * setup bug and aborts the scheduler. On a CPU outside the scheduler's cid + * space the helpers fall back to the first matching cid. + * + * TODO: The slots are one allocation without node placement. Move them to a + * per-cid arena allocator once the library has one, for node-local slots + * without the cacheline padding. + */ +struct cmask_rotor_slot { + u32 cid; +} __attribute__((aligned(SCX_CACHELINE_SIZE))); + +struct cmask_rotor_slot __arena *cmask_distribute_rotors __weak; + +/** + * cmask_distribute_init - Allocate the distribute rotor from @map + * @map: the scheduler's arena map + * + * The shared arena init calls this once. A scheduler with its own arena calls + * it before the first distribute pick. On a kernel without the cid kfuncs + * nothing can pick, so the call allocates nothing and succeeds. Return 0 on + * success, -ENOMEM if the allocation fails. + */ +static __always_inline int cmask_distribute_init(void *map) +{ + u64 size; + u32 pages; + + /* no cid kfuncs means no picks and nothing to allocate */ + if (!bpf_ksym_exists(scx_bpf_nr_cids)) + return 0; + + size = scx_bpf_nr_cids() * sizeof(*cmask_distribute_rotors); + pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + cmask_distribute_rotors = bpf_arena_alloc_pages(map, NULL, pages, NUMA_NO_NODE, 0); + return cmask_distribute_rotors ? 0 : -ENOMEM; +} + +static __always_inline u32 __arena *__cmask_distribute_rotor(void) +{ + s32 cid = scx_bpf_this_cid(); + + if (unlikely(!cmask_distribute_rotors)) { + scx_bpf_error("cmask distribute rotor not allocated"); + return NULL; + } + if (cid < 0) + return NULL; + return &cmask_distribute_rotors[cid].cid; +} + +/** + * cmask_any_distribute - Pick a set cid, spreading successive picks + * @m: cmask to pick from + * + * Counterpart of bpf_cpumask_any_distribute(): a per-cid rotor makes successive + * picks rotate through the set cids instead of repeating the first one. Returns + * cmask_end(@m) if @m is empty. + */ +static __always_inline u32 cmask_any_distribute(const struct scx_cmask __arena *m) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + u32 pick; + + if (unlikely(!rotor)) + return cmask_first_set(m); + + pick = cmask_next_set_wrap(m, *rotor + 1); + if (pick < cmask_end(m)) + *rotor = pick; + return pick; +} + +/** + * cmask_any_and_distribute - Pick a cid set in both masks, spreading picks + * @a: first cmask, the scan is bounded by its range + * @b: second cmask + * + * Counterpart of bpf_cpumask_any_and_distribute(). Shares the rotor with + * cmask_any_distribute() the same way the kernel counterparts share theirs. + * Returns cmask_end(@a) if the intersection is empty. + */ +static __always_inline u32 cmask_any_and_distribute(const struct scx_cmask __arena *a, + const struct scx_cmask __arena *b) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + u32 pick; + + if (unlikely(!rotor)) + return cmask_next_and_set(a, b, a->base); + + pick = cmask_next_and_set_wrap(a, b, *rotor + 1); + if (pick < cmask_end(a)) + *rotor = pick; + return pick; +} + +/** + * cmask_distribute_rotor_pos - Last cid picked by the distribute helpers + * + * For callers that anchor scans of their own on the shared rotor. Returns 0 + * when nothing has been picked on this cid yet or the CPU is outside the cid + * space. + */ +static __always_inline u32 cmask_distribute_rotor_pos(void) +{ + u32 __arena *rotor = __cmask_distribute_rotor(); + + if (unlikely(!rotor)) + return 0; + return *rotor; +} + /* * Like cmask_next_and_set() but over the intersection of THREE masks. Return * a->base + a->nr_cids if no cid is set in all three at or after @start. @@ -701,7 +1202,7 @@ static __always_inline u32 cmask_next_and2_set(const struct scx_cmask __arena *a start_wi = start / 64; start_bit = start & 63; - bpf_for(i, 0, CMASK_MAX_WORDS) { + bpf_arena_for(i, 0, CMASK_MAX_WORDS) { u32 abs_wi = start_wi + i; u64 word; u32 found; @@ -763,7 +1264,7 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m, s32 cpu; cmask_zero(m); - bpf_for(cpu, 0, nr_cpu_ids) { + bpf_arena_for(cpu, 0, nr_cpu_ids) { s32 cid; if (!bpf_cpumask_test_cpu(cpu, cpumask)) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 5003e33f5142..deeae74ee5ff 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -27,6 +27,7 @@ #include "user_exit_info.bpf.h" #include "enum_defs.autogen.h" #include "bpf_arena_common.bpf.h" +#include "const-defs.h" #define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ @@ -107,8 +108,7 @@ void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __ s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; s32 scx_bpf_cid_node(s32 cid) __ksym __weak; -void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak; -void scx_bpf_kick_cid(s32 cid, u64 flags) __ksym __weak; +/* scx_bpf_cid_topo() and scx_bpf_kick_cid() are declared in compat.bpf.h */ s32 scx_bpf_task_cid(const struct task_struct *p) __ksym __weak; s32 scx_bpf_this_cid(void) __ksym __weak; struct task_struct *scx_bpf_cid_curr(s32 cid) __ksym __weak; @@ -117,7 +117,7 @@ u32 scx_bpf_nr_online_cids(void) __ksym __weak; const void __arena *scx_bpf_online_cmask(void) __ksym __weak; u32 scx_bpf_cidperf_cap(s32 cid) __ksym __weak; u32 scx_bpf_cidperf_cur(s32 cid) __ksym __weak; -s32 scx_bpf_cidperf_set(s32 cid, u32 perf) __ksym __weak; +/* scx_bpf_cidperf_set() is declared in compat.bpf.h */ /* sub-scheduler cap control, scx_bpf_sub_caps() cgroup_id 0 == self */ s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask __arena *cmask__arena, struct scx_cmask __arena *denied_out__arena__nullable) __ksym __weak; @@ -534,14 +534,14 @@ static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) /* * True if the non-sleepable BPF trampoline prolog (__bpf_prog_enter) calls * migrate_disable() for the current task. Recorded once by - * scx_lib_init_probe, an fentry program on bpf_scx_reg() that fires during - * the natural scheduler-attach call chain (auto-attached by scx_ops_attach!). + * scx_lib_init_probe, an fentry program that fires during the natural + * scheduler-attach call chain (auto-attached by scx_ops_attach!). * - * Defaults to true (conservative). Over-reporting in is_migration_disabled() + * Defaults to false (conservative). Over-reporting in is_migration_disabled() * causes local-only dispatch, which is safe. Under-reporting can crash the * scheduler, so we err high if the probe somehow fails to run. */ -bool __scx_prolog_disables_migration __weak = true; +bool __scx_prolog_disables_migration __weak = false; /* * scx_lib_init_probe - non-sleepable prolog probe. @@ -552,13 +552,20 @@ bool __scx_prolog_disables_migration __weak = true; * ops.init() fires. Its address is taken in the vtable, so the symbol * is non-inlinable and has been stable since introduction. * + * A cid-form scheduler registers through bpf_scx_reg_cid(), the .reg + * callback of bpf_sched_ext_ops_cid, and so never enters bpf_scx_reg(). + * The cid-form open paths (SCX_OPS_CID_OPEN(), scx_ops_cid_open!()) + * therefore repoint this program at bpf_scx_reg_cid(). Repointing rather + * than adding a second program keeps the object loadable on kernels + * predating the cid form, where bpf_scx_reg_cid() has no BTF entry. + * * Entering via fentry runs us through __bpf_prog_enter -- the * non-sleepable prolog that consumers of is_migration_disabled() live * under. * * Loud warning: the prolog adds at most 1 to migration_disabled. * Reading > 1 means something upstream in the - * bpf_struct_ops_link_create -> bpf_scx_reg path disabled migration + * bpf_struct_ops_link_create -> .reg path disabled migration * before the prolog ran, invalidating the probe; audit and adjust. */ SEC("fentry/bpf_scx_reg") __weak @@ -1186,8 +1193,75 @@ static inline u64 scx_clock_irq(u32 cpu) #define struct_size_t(type, member, count) \ struct_size((type *)NULL, member, count) +/* + * 's TRAILING_OVERLAP(): embed a struct that ends in a flexible + * array member together with its storage. @NAME is a complete @TYPE and the + * @MEMBERS declared after the array reserve the space the array grows into. + * Unlike the kernel's, the padding is named after @NAME so that one struct can + * embed several, and it is sized with __builtin_offsetof() because + * bpf_helpers.h redefines offsetof() as a pointer cast, which is not a constant + * expression. + */ +#define __TRAILING_OVERLAP(TYPE, NAME, FAM, ATTRS, MEMBERS) \ + union { \ + TYPE NAME; \ + struct { \ + unsigned char __offset_to_##NAME[__builtin_offsetof(TYPE, FAM)]; \ + MEMBERS \ + } ATTRS; \ + } + +#define TRAILING_OVERLAP(TYPE, NAME, FAM, MEMBERS) \ + __TRAILING_OVERLAP(TYPE, NAME, FAM, /* no attrs */, MEMBERS) + +/* + * Loop counters the verifier cannot see through. + * + * A may_goto loop converges when the state at its head is within the state + * of a previous iteration. A counter tracked as a precise constant prevents + * that convergence: every iteration creates a new state, and the verifier + * unrolls the loop until it runs out of budget whenever the counter feeds an + * operation that requires precision. + * + * Loading a writable global gives the verifier an unknown scalar while its + * runtime value remains one. Using it as the step makes the counter unknown + * after the first iteration, and the load on every increment keeps the loop + * body from making it precise again. volatile is required to keep the compiler + * from hoisting or eliminating the loads, and the value must remain non-const + * so the verifier cannot resolve it. The variable is weak so the objects linked + * into a scheduler share one copy. Its runtime value must never be changed from + * one. + * + * bpf_for() avoids this verifier behavior too, but calls bpf_iter_num_next() + * on every iteration. bpf_arena_for() is intended for hot scheduler walks + * where that cost matters. @var must be no wider than u32, and @start and + * @end must be representable as u32. + */ +volatile u32 bpf_arena_loop_one __weak = 1; + +/* + * A loop-carried accumulator that a caller later compares is kept precise, and + * a precise scalar whose range grows every iteration keeps a may_goto loop from + * converging. Initializing it from this zero makes it unknown from the first + * iteration, so every pass through the loop head looks the same. + */ +volatile u32 bpf_arena_loop_zero __weak = 0; + +#define __bpf_arena_loop_start(var, start) \ + ({ \ + _Static_assert(sizeof(var) <= sizeof(u32), \ + "bpf_arena_for() index must fit in u32"); \ + (start); \ + }) + +#define bpf_arena_for(var, start, end) \ + for (var = __bpf_arena_loop_start(var, start); \ + var < (end) && can_loop; \ + var += bpf_arena_loop_one) + #include "compat.bpf.h" #include "enums.bpf.h" +#include "features.bpf.h" #include "cid.bpf.h" #endif /* __SCX_COMMON_BPF_H */ diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index c5c90d1d81e0..f5726bbfe91b 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -143,6 +143,69 @@ static inline void scx_bpf_cid_override(const s32 __arena *cpu_to_cid, u32 cpu_t shard_start, shard_start_cnt); } +/* + * v7.3: b38332be61a8 ("sched_ext: Make scx_bpf_kick_cid() return void") flipped + * the return type from s32 to void without renaming the kfunc, so the v7.2 and + * v7.3 kfuncs share a name but have incompatible prototypes. libbpf leaves a + * weak kfunc unresolved when the prototype doesn't match, so declare both + * flavors: the ___vN suffix is stripped on the BPF side, both resolve against + * the same kernel symbol and only the matching one gets bound. Drop the + * wrapper and move the decl back to common.bpf.h after v7.6. + */ +void scx_bpf_kick_cid___v2(s32 cid, u64 flags) __ksym __weak; +s32 scx_bpf_kick_cid___v1(s32 cid, u64 flags) __ksym __weak; + +static inline void scx_bpf_kick_cid(s32 cid, u64 flags) +{ + if (bpf_ksym_exists(scx_bpf_kick_cid___v2)) + scx_bpf_kick_cid___v2(cid, flags); + else if (bpf_ksym_exists(scx_bpf_kick_cid___v1)) + scx_bpf_kick_cid___v1(cid, flags); +} + +/* + * v7.3: 94480606a677 ("sched_ext: Add a size argument to scx_bpf_cid_topo() so + * struct scx_cid_topo can grow") added out__sz without renaming the kfunc, so + * the v7.2 and v7.3 kfuncs share a name with incompatible prototypes. Same + * two-flavor trick as scx_bpf_kick_cid() above. The size is always the output + * buffer's, so a macro supplies it and callers keep the two-argument form. + * Drop the wrapper and move the decl back to common.bpf.h after v7.6. + */ +void scx_bpf_cid_topo___v2(s32 cid, struct scx_cid_topo *out, size_t out__sz) __ksym __weak; +void scx_bpf_cid_topo___v1(s32 cid, struct scx_cid_topo *out) __ksym __weak; + +static inline void __scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out, size_t out__sz) +{ + if (bpf_ksym_exists(scx_bpf_cid_topo___v2)) + scx_bpf_cid_topo___v2(cid, out, out__sz); + else if (bpf_ksym_exists(scx_bpf_cid_topo___v1)) + scx_bpf_cid_topo___v1(cid, out); +} + +#define scx_bpf_cid_topo(cid, out) __scx_bpf_cid_topo((cid), (out), sizeof(*(out))) + +/* + * v7.3: 3a21e34eb258 ("sched_ext: Gate scx_bpf_cidperf_set() behind a new + * SCX_CAP_PERF") flipped the return type the other way, from void to s32, again + * without renaming the kfunc. Same two-flavor trick as scx_bpf_kick_cid() + * above; the v7.2 kfunc can't fail, so report success for it. Drop the wrapper + * and move the decl back to common.bpf.h after v7.6. + */ +s32 scx_bpf_cidperf_set___v2(s32 cid, u32 perf) __ksym __weak; +void scx_bpf_cidperf_set___v1(s32 cid, u32 perf) __ksym __weak; + +static inline s32 scx_bpf_cidperf_set(s32 cid, u32 perf) +{ + if (bpf_ksym_exists(scx_bpf_cidperf_set___v2)) { + return scx_bpf_cidperf_set___v2(cid, perf); + } else if (bpf_ksym_exists(scx_bpf_cidperf_set___v1)) { + scx_bpf_cidperf_set___v1(cid, perf); + return 0; + } else { + return -EOPNOTSUPP; + } +} + /** * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on * in a compatible way. We will preserve this __COMPAT helper until v6.16. diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index 58253fe8e85d..07fb55e63c57 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -336,9 +336,20 @@ static inline long scx_hotplug_seq(void) /* * Open a cid-form (struct sched_ext_ops_cid) skeleton. The cid form postdates * every op the load-time fix-ups above handle, so none of them apply. + * + * A cid-form scheduler registers through bpf_scx_reg_cid() rather than + * bpf_scx_reg(), so repoint common.bpf.h's prolog probe at it. Both the cid + * form and bpf_scx_reg_cid() appeared in v7.2, so a kernel that accepts this + * skeleton always has the symbol. */ -#define SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, __opts) \ - __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid", __opts) +#define SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, __opts) ({ \ + struct __scx_name *__cskel; \ + \ + __cskel = __SCX_OPS_OPEN(__ops_name, __scx_name, "sched_ext_ops_cid", __opts); \ + bpf_program__set_attach_target(__cskel->progs.scx_lib_init_probe, 0, \ + "bpf_scx_reg_cid"); \ + __cskel; \ +}) #define SCX_OPS_CID_OPEN(__ops_name, __scx_name) \ SCX_OPS_CID_OPEN_OPTS(__ops_name, __scx_name, 0) diff --git a/tools/sched_ext/include/scx/const-defs.h b/tools/sched_ext/include/scx/const-defs.h new file mode 100644 index 000000000000..4c8f45756b51 --- /dev/null +++ b/tools/sched_ext/include/scx/const-defs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Tejun Heo */ +#pragma once + +/* + * Constants shared across the BPF headers, the lib and the schedulers. + * Freestanding so that scheduler interface headers consumed by bindgen can + * include it. Rust code cannot, so rust/scx_arena/scx_arena/src/lib.rs mirrors + * the values. Keep them in sync. + */ +enum scx_const_defs { + /* mirrors the kernel's per-arch L1_CACHE_SHIFT */ +#if defined(__TARGET_ARCH_s390) || defined(__s390x__) + SCX_CACHELINE_SIZE = 256, +#elif defined(__TARGET_ARCH_powerpc) || defined(__powerpc64__) + SCX_CACHELINE_SIZE = 128, +#else + SCX_CACHELINE_SIZE = 64, +#endif +}; diff --git a/tools/sched_ext/include/scx/features.bpf.h b/tools/sched_ext/include/scx/features.bpf.h new file mode 100644 index 000000000000..992f4a74b277 --- /dev/null +++ b/tools/sched_ext/include/scx/features.bpf.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Kernel features the loader probes before load and hands to the BPF side in + * one rodata word, so that a program can carry both the path a feature enables + * and its fallback while the verifier prunes the one the running kernel cannot + * take. scx_utils sets the word in the ops open macros. The bits are mirrored + * in rust/scx_utils/src/compat.rs. + * + * Included by scx/common.bpf.h; don't include directly. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Tejun Heo + */ +#ifndef __SCX_FEATURES_BPF_H +#define __SCX_FEATURES_BPF_H + +enum scx_lib_feature { + /* the JIT lowers fetching AND, OR and XOR on arena pointers */ + SCX_LIB_FEAT_ARENA_FETCH_BITOPS = 1ULL << 0, +}; + +const volatile u64 scx_lib_features __weak; + +static __always_inline bool scx_lib_has(u64 feat) +{ + return scx_lib_features & feat; +} + +#endif /* __SCX_FEATURES_BPF_H */ diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 81575294027e..3566e3e02e21 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1473,7 +1473,8 @@ __noinline void compute_partition(void) /* find out the cids we hold */ scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); - cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); /* held only as ENQ_IMMED */ + /* held only as ENQ_IMMED */ + cmask_andnot(&qa.held_shared.mask, &qa.held_shared.mask, &qa.held_excl.mask); qa.part.nr_shared = 0; qa.part.nr_rr = 0; @@ -1628,8 +1629,7 @@ static __noinline void account_alloc(void) */ static void refresh_usable(void) { - cmask_copy(&qa.usable_scratch.mask, &qa.self_cids.mask); - cmask_and(&qa.usable_scratch.mask, &qa.avail_cids.mask); + cmask_and(&qa.usable_scratch.mask, &qa.self_cids.mask, &qa.avail_cids.mask); cmask_copy(&qa.usable_cids.mask, &qa.usable_scratch.mask); } @@ -1708,10 +1708,8 @@ __noinline void apply_partition(void) if (!cgid) continue; - cmask_copy(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask); - cmask_andnot(&qa.to_revoke_cids.mask, &ssc->granted_cids.mask); - cmask_copy(&qa.to_grant_cids.mask, &ssc->granted_cids.mask); - cmask_andnot(&qa.to_grant_cids.mask, &ssc->prev_granted.mask); + cmask_andnot(&qa.to_revoke_cids.mask, &ssc->prev_granted.mask, &ssc->granted_cids.mask); + cmask_andnot(&qa.to_grant_cids.mask, &ssc->granted_cids.mask, &ssc->prev_granted.mask); scx_bpf_sub_revoke(cgid, SCX_CAP_ENQ_IMMED | SCX_CAP_PERF, &qa.prev_rr_cids.mask); @@ -1957,7 +1955,7 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) scx_bpf_sub_caps(0, SCX_CAP_ENQ, &qa.held_excl.mask); scx_bpf_sub_caps(0, SCX_CAP_ENQ_IMMED, &qa.held_shared.mask); - cmask_andnot(&qa.held_shared.mask, &qa.held_excl.mask); + cmask_andnot(&qa.held_shared.mask, &qa.held_shared.mask, &qa.held_excl.mask); bpf_for(i, 0, MAX_SUB_SCHEDS) { cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, nr_cids); From da0946effa09ea937d971b6fab2d6e380048dbb4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 29 Sep 2026 13:20:29 -1000 Subject: [PATCH 44/69] sched_ext: Sync tools autogen enum headers from the scx repo Sync the four autogen enum headers with the scx repo at c9b90b3ad3d0 ("scheds/include: Sync with kernel sched_ext/for-7.4 (c6fe97c34a1a)"). The enum definition and ABI tables pick up the enumerators the kernel gained since the last sync. The two lookup tables are taken verbatim from scx's copies so the files stay identical: their task state lookups move from the removed scx_task_state to scx_ent_flags, where 7203d77d6e04 ("sched_ext: Simplify task state handling") put the states, and until now resolved to zero. The only entry the kernel copy gains is SCX_RQ_BAL_KEEP, which scx keeps for older kernels and which zero-fills here. Signed-off-by: Tejun Heo --- .../sched_ext/include/scx/enum_defs.autogen.h | 3 +++ .../sched_ext/include/scx/enums.autogen.bpf.h | 15 ++++++++------ tools/sched_ext/include/scx/enums.autogen.h | 20 ++++++++++--------- .../sched_ext/include/scx/enums_abi.autogen.h | 6 ++++++ 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 9707abeee00d..4fa82033b99c 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -171,6 +171,7 @@ #define HAVE_SCX_OPS_ALWAYS_ENQ_IMMED #define HAVE_SCX_OPS_TID_TO_TASK #define HAVE_SCX_OPS_LAZY_RESCHED +#define HAVE_SCX_OPS_ENQ_BLOCKED #define HAVE_SCX_OPS_ALL_FLAGS #define HAVE___SCX_OPS_INTERNAL_MASK #define HAVE_SCX_OPS_HAS_CPU_PREEMPT @@ -198,6 +199,8 @@ #define HAVE_SCX_RQ_BAL_CB_PENDING #define HAVE_SCX_RQ_SUB_IDLE_RENOTIFY #define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY +#define HAVE_SCX_RQ_PROXY_RETRY +#define HAVE_SCX_RQ_PROXY_TICK #define HAVE_SCX_RQ_IN_WAKEUP #define HAVE_SCX_RQ_IN_DISPATCH #define HAVE_SCX_SCHED_PCPU_BYPASSING diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h index 0a5478afbe79..405a3e6b8f33 100644 --- a/tools/sched_ext/include/scx/enums.autogen.bpf.h +++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h @@ -1,9 +1,9 @@ /* - * WARNING: This file is autogenerated from scripts/gen_enums.py. If you would - * like to access an enum that is currently missing, add it to the script - * and run it from the root directory to update this file. + * Definitions for SCX constants. Keep this table in sync + * with rust/scx_utils/src/enums.rs. */ + const volatile u64 __SCX_OPS_NAME_LEN __weak; #define SCX_OPS_NAME_LEN __SCX_OPS_NAME_LEN @@ -22,6 +22,9 @@ const volatile u64 __SCX_RQ_CAN_STOP_TICK __weak; const volatile u64 __SCX_RQ_BAL_PENDING __weak; #define SCX_RQ_BAL_PENDING __SCX_RQ_BAL_PENDING +const volatile u64 __SCX_RQ_BAL_KEEP __weak; +#define SCX_RQ_BAL_KEEP __SCX_RQ_BAL_KEEP + const volatile u64 __SCX_RQ_BYPASSING __weak; #define SCX_RQ_BYPASSING __SCX_RQ_BYPASSING @@ -136,15 +139,15 @@ const volatile u64 __SCX_ENQ_IMMED __weak; const volatile u64 __SCX_ENQ_RESCUE __weak; #define SCX_ENQ_RESCUE __SCX_ENQ_RESCUE +const volatile u64 __SCX_ENQ_BLOCKED __weak; +#define SCX_ENQ_BLOCKED __SCX_ENQ_BLOCKED + const volatile u64 __SCX_ENQ_REENQ __weak; #define SCX_ENQ_REENQ __SCX_ENQ_REENQ const volatile u64 __SCX_ENQ_LAST __weak; #define SCX_ENQ_LAST __SCX_ENQ_LAST -const volatile u64 __SCX_ENQ_BLOCKED __weak; -#define SCX_ENQ_BLOCKED __SCX_ENQ_BLOCKED - const volatile u64 __SCX_ENQ_CLEAR_OPSS __weak; #define SCX_ENQ_CLEAR_OPSS __SCX_ENQ_CLEAR_OPSS diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h index 60a1e5576126..7ec2e5e41cb2 100644 --- a/tools/sched_ext/include/scx/enums.autogen.h +++ b/tools/sched_ext/include/scx/enums.autogen.h @@ -1,7 +1,8 @@ +#pragma once + /* - * WARNING: This file is autogenerated from scripts/gen_enums.py. If you would - * like to access an enum that is currently missing, add it to the script - * and run it from the root directory to update this file. + * BTF enum lookups for SCX constants. Keep this table in sync + * with rust/scx_utils/src/enums.rs. */ #define SCX_ENUM_INIT(skel) do { \ @@ -11,6 +12,7 @@ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_ONLINE); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CAN_STOP_TICK); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_PENDING); \ + SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BAL_KEEP); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_BYPASSING); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_CLK_VALID); \ SCX_ENUM_SET(skel, scx_rq_flags, SCX_RQ_IN_WAKEUP); \ @@ -32,11 +34,11 @@ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_BITS); \ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_STATE_MASK); \ SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_CURSOR); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NONE); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_INIT); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_READY); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_ENABLED); \ - SCX_ENUM_SET(skel, scx_task_state, SCX_TASK_NR_STATES); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_NONE); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_INIT); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_READY); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_ENABLED); \ + SCX_ENUM_SET(skel, scx_ent_flags, SCX_TASK_NR_STATES); \ SCX_ENUM_SET(skel, scx_ent_dsq_flags, SCX_TASK_DSQ_ON_PRIQ); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_IDLE); \ SCX_ENUM_SET(skel, scx_kick_flags, SCX_KICK_PREEMPT); \ @@ -49,9 +51,9 @@ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT_LAZY); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_IMMED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \ + SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_BLOCKED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \ - SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_BLOCKED); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_DSQ_PRIQ); \ SCX_ENUM_SET(skel, scx_deq_flags, SCX_DEQ_SCHED_CHANGE); \ diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index 251eea9b81e7..1186daa9cb14 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -102,6 +102,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_enq_flags", "SCX_ENQ_RESCUE", 0x400000000LLU }, { "scx_enq_flags", "SCX_ENQ_REENQ", 0x10000000000LLU }, { "scx_enq_flags", "SCX_ENQ_LAST", 0x20000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_BLOCKED", 0x40000000000LLU }, { "scx_enq_flags", "__SCX_ENQ_INTERNAL_MASK", 0xff00000000000000LLU }, { "scx_enq_flags", "SCX_ENQ_CLEAR_OPSS", 0x100000000000000LLU }, { "scx_enq_flags", "SCX_ENQ_DSQ_PRIQ", 0x200000000000000LLU }, @@ -118,6 +119,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_SUB_INIT", 0x10LLU }, { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, + { "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU }, { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU }, { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU }, @@ -135,6 +137,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU }, { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU }, { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x5000LLU }, { "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU }, @@ -180,6 +183,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU }, { "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU }, { "scx_ops_flags", "SCX_OPS_LAZY_RESCHED", 0x200LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_BLOCKED", 0x400LLU }, { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x7ffLLU }, { "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU }, { "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU }, @@ -207,6 +211,8 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_rq_flags", "SCX_RQ_BAL_CB_PENDING", 0x40LLU }, { "scx_rq_flags", "SCX_RQ_SUB_IDLE_RENOTIFY", 0x80LLU }, { "scx_rq_flags", "SCX_RQ_ROOT_IDLE_RENOTIFY", 0x100LLU }, + { "scx_rq_flags", "SCX_RQ_PROXY_RETRY", 0x200LLU }, + { "scx_rq_flags", "SCX_RQ_PROXY_TICK", 0x400LLU }, { "scx_rq_flags", "SCX_RQ_IN_WAKEUP", 0x10000LLU }, { "scx_rq_flags", "SCX_RQ_IN_DISPATCH", 0x20000LLU }, { "scx_sched_pcpu_flags", "SCX_SCHED_PCPU_BYPASSING", 0x1LLU }, From e28807c846081ab04ba5eb77e588878ee3811961 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 30 Sep 2026 11:00:18 +0200 Subject: [PATCH 45/69] sched_ext: Introduce scx_bpf_cgroup_nr_cpus() Hierarchical BPF schedulers may need to know how many CPUs a cgroup can run on when distributing group-wide weights. For example, fair's group share calculation scales a group's weight by the smaller of its estimated runnable task count and the number of CPUs in its effective cpuset. Without that count, a BPF scheduler can only use the machine-wide CPU count and may give a group confined to a few CPUs excessive shares. A task's effective affinity can include per-task restrictions, so it cannot reliably represent the group's cpuset. Reading the cpuset directly from BPF would require CO-RE access to private cgroup and cpuset structures, including inherited cpuset resolution, making the approach fragile across kernel versions. Add scx_bpf_cgroup_nr_cpus(), which returns the same count fair uses through cpuset_num_cpus(). The count is independent of CPU or CID numbering, so the kfunc is available to both cpu-form and cid-form schedulers from any context. Also add a compat wrapper which falls back to nr_cpu_ids on older kernels. Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 18 ++++++++++++++++++ tools/sched_ext/include/scx/common.bpf.h | 1 + tools/sched_ext/include/scx/compat.bpf.h | 7 +++++++ 3 files changed, 26 insertions(+) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 0ef48c2ba6c8..96c904b39601 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -11377,6 +11377,23 @@ __bpf_kfunc struct cgroup *scx_bpf_task_cgroup(struct task_struct *p, cgroup_get(cgrp); return cgrp; } + +/** + * scx_bpf_cgroup_nr_cpus - Return the number of CPUs in a cgroup's cpuset + * @cgrp: cgroup of interest + * + * Return the number of CPUs in @cgrp's effective cpuset, which is inherited + * from the nearest ancestor with the cpuset controller enabled. This matches + * the count used by fair's group share calculation and lets hierarchical BPF + * schedulers bound a group's weight by the CPUs it can actually run on. + * + * The value is a snapshot and may change at any time through cpuset updates or + * CPU hotplug. Schedulers should re-read it when recomputing group state. + */ +__bpf_kfunc u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) +{ + return cpuset_num_cpus(cgrp); +} #endif /* CONFIG_CGROUP_SCHED */ __bpf_kfunc_end_defs(); @@ -11424,6 +11441,7 @@ BTF_ID_FLAGS(func, scx_bpf_now) BTF_ID_FLAGS(func, scx_bpf_events, KF_IMPLICIT_ARGS) #ifdef CONFIG_CGROUP_SCHED BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_IMPLICIT_ARGS | KF_RCU | KF_ACQUIRE) +BTF_ID_FLAGS(func, scx_bpf_cgroup_nr_cpus, KF_RCU) #endif BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index deeae74ee5ff..651bd4952647 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -105,6 +105,7 @@ struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; u64 scx_bpf_now(void) __ksym __weak; void scx_bpf_events(struct scx_event_stats *events, size_t events__sz) __ksym __weak; +u32 scx_bpf_cgroup_nr_cpus(struct cgroup *cgrp) __ksym __weak; s32 scx_bpf_cpu_to_cid(s32 cpu) __ksym __weak; s32 scx_bpf_cid_to_cpu(s32 cid) __ksym __weak; s32 scx_bpf_cid_node(s32 cid) __ksym __weak; diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index f5726bbfe91b..c26024a6e2b1 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -311,6 +311,13 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) (bpf_ksym_exists(scx_bpf_cid_node) ? \ scx_bpf_cid_node(cid) : NUMA_NO_NODE) +/* + * v7.4: Add scx_bpf_cgroup_nr_cpus(). + */ +#define __COMPAT_scx_bpf_cgroup_nr_cpus(cgrp) \ + (bpf_ksym_exists(scx_bpf_cgroup_nr_cpus) ? \ + scx_bpf_cgroup_nr_cpus(cgrp) : scx_bpf_nr_cpu_ids()) + /* * v6.18: Add a helper to retrieve the current task running on a CPU. * From 3521f92ddd5f1a397e6ed9a6727694ca85950fb3 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 30 Sep 2026 11:00:19 +0200 Subject: [PATCH 46/69] selftests/sched_ext: Test scx_bpf_cgroup_nr_cpus() Verify that scx_bpf_cgroup_nr_cpus() reports the number of CPUs in a cgroup's effective cpuset. Create a cgroup-v2 subtree where a child owns a cpuset and a leaf inherits it. Sample the kfunc from a BPF_PROG_TYPE_SYSCALL program and compare it with cpuset.cpus.effective for the root, the cpuset owner and the inheriting leaf, without a scheduler attached. Attach a scheduler and verify the value observed from ops.cgroup_init(). Then restrict the child's cpuset to a non-contiguous and a single-CPU set, and finally disable cpuset below the parent so that both cgroups inherit the parent's cpuset. A disabled cpuset css stays attached to its cgroup until it is asynchronously offlined, so poll briefly for the last step. Skip the test when the required controllers or two CPUs are unavailable. Assisted-by: LLM Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/Makefile | 6 +- .../selftests/sched_ext/cgroup_nr_cpus.bpf.c | 60 +++ .../selftests/sched_ext/cgroup_nr_cpus.c | 419 ++++++++++++++++++ tools/testing/selftests/sched_ext/config | 1 + 4 files changed, 484 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c create mode 100644 tools/testing/selftests/sched_ext/cgroup_nr_cpus.c diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index ff0ccf71f30c..c5d3a2eaea7d 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -10,6 +10,7 @@ TEST_GEN_MODS_DIR := test_modules # override lib.mk's default rules OVERRIDE_TARGETS := 1 include ../lib.mk +include ../cgroup/lib/libcgroup.mk CURDIR := $(abspath .) REPOROOT := $(abspath ../../../..) @@ -154,7 +155,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP override define CLEAN rm -rf $(OUTPUT_DIR) - rm -f $(TEST_GEN_PROGS) + rm -f $(TEST_GEN_PROGS) $(EXTRA_CLEAN) endef # Every testcase takes all of the BPF progs are dependencies by default. This @@ -163,6 +164,7 @@ endef all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubst %.c,%.skel.h,$(prog))) auto-test-targets := \ + cgroup_nr_cpus \ create_dsq \ dequeue \ dequeue_iter \ @@ -217,7 +219,7 @@ $(testcase-targets): $(SCXOBJ_DIR)/%.o: %.c $(SCXOBJ_DIR)/runner.o $(all_test_bp $(SCXOBJ_DIR)/util.o: util.c | $(SCXOBJ_DIR) $(CC) $(CFLAGS) -c $< -o $@ -$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(testcase-targets) +$(OUTPUT)/runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(LIBCGROUP_O) $(testcase-targets) @echo "$(testcase-targets)" $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) diff --git a/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c new file mode 100644 index 000000000000..841c83abb41b --- /dev/null +++ b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.bpf.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Validate scx_bpf_cgroup_nr_cpus() from both BPF_PROG_TYPE_SYSCALL and + * struct_ops contexts. + * + * Copyright (c) 2026 NVIDIA Corporation. + */ + +#include + +char _license[] SEC("license") = "GPL"; + +UEI_DEFINE(uei); + +/* input to cgroup_nr_cpus_read() */ +u64 query_cgid; +/* output of cgroup_nr_cpus_read(), -1 if @query_cgid couldn't be resolved */ +s64 query_nr_cpus = -1; + +/* recorded by ops.cgroup_init() for @init_cgid */ +u64 init_cgid; +s64 init_nr_cpus = -1; + +SEC("syscall") +int cgroup_nr_cpus_read(void *ctx) +{ + struct cgroup *cgrp; + + query_nr_cpus = -1; + + cgrp = bpf_cgroup_from_id(query_cgid); + if (!cgrp) + return -ENOENT; + + query_nr_cpus = scx_bpf_cgroup_nr_cpus(cgrp); + bpf_cgroup_release(cgrp); + + return 0; +} + +s32 BPF_STRUCT_OPS(cgroup_nr_cpus_cgroup_init, struct cgroup *cgrp, + struct scx_cgroup_init_args *args) +{ + if (cgrp->kn->id == init_cgid) + init_nr_cpus = scx_bpf_cgroup_nr_cpus(cgrp); + + return 0; +} + +void BPF_STRUCT_OPS(cgroup_nr_cpus_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + +SEC(".struct_ops.link") +struct sched_ext_ops cgroup_nr_cpus_ops = { + .cgroup_init = (void *)cgroup_nr_cpus_cgroup_init, + .exit = (void *)cgroup_nr_cpus_exit, + .name = "cgroup_nr_cpus", +}; diff --git a/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c new file mode 100644 index 000000000000..4973f7080353 --- /dev/null +++ b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Verify that scx_bpf_cgroup_nr_cpus() reports the number of CPUs in a + * cgroup's effective cpuset, including inherited and updated cpusets. + * + * Copyright (c) 2026 NVIDIA Corporation. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cgroup_nr_cpus.bpf.skel.h" +#include "cgroup_util.h" +#include "scx_test.h" + +/* + * Hierarchy under the cgroup2 root, all with the cpu controller enabled so + * that ops.cgroup_init() runs for each of them: + * + * parent cpuset enabled by the root, enables cpuset for its children + * parent/child owns a cpuset + * parent/child/leaf no cpuset of its own, inherits child's + */ +struct cgroup_nr_cpus_ctx { + struct cgroup_nr_cpus *skel; + struct bpf_link *link; + char root[PATH_MAX]; + char parent[PATH_MAX]; + char child[PATH_MAX]; + char leaf[PATH_MAX]; + bool parent_created; + bool child_created; + bool leaf_created; +}; + +static int join_path(char *dst, size_t dst_size, const char *parent, const char *name) +{ + int ret; + + ret = snprintf(dst, dst_size, "%s/%s", parent, name); + if (ret < 0 || (size_t)ret >= dst_size) + return -ENAMETOOLONG; + return 0; +} + +static u64 cgroup_id(const char *path) +{ + union { + u64 id; + unsigned char bytes[8]; + } id = {}; + struct file_handle *handle; + int mount_id, ret; + + handle = calloc(1, sizeof(*handle) + sizeof(id)); + if (!handle) + return 0; + handle->handle_bytes = sizeof(id); + ret = name_to_handle_at(AT_FDCWD, path, handle, &mount_id, 0); + if (!ret && handle->handle_bytes == sizeof(id)) + memcpy(id.bytes, handle->f_handle, sizeof(id)); + free(handle); + + return ret ? 0 : id.id; +} + +/* + * Parse a cpulist such as "0-3,8,10-11". Return the number of CPUs and the + * lowest and highest CPU in @first and @last, or -errno on failure. + */ +static int parse_cpulist(const char *cpulist, u32 *first, u32 *last) +{ + const char *p = cpulist; + u32 lowest = UINT_MAX, highest = 0; + int count = 0; + + while (*p && *p != '\n') { + unsigned long start, end_cpu; + char *end; + + errno = 0; + start = strtoul(p, &end, 10); + if (errno || end == p || start > INT_MAX) + return -EINVAL; + end_cpu = start; + p = end; + if (*p == '-') { + end_cpu = strtoul(p + 1, &end, 10); + if (errno || end == p + 1 || end_cpu > INT_MAX || end_cpu < start) + return -EINVAL; + p = end; + } + if (end_cpu - start + 1 > (unsigned long)(INT_MAX - count)) + return -EOVERFLOW; + if (start < lowest) + lowest = start; + if (end_cpu > highest) + highest = end_cpu; + count += end_cpu - start + 1; + if (*p == ',') + p++; + else if (*p && *p != '\n') + return -EINVAL; + } + + if (first) + *first = lowest; + if (last) + *last = highest; + return count; +} + +/* + * Number of CPUs in @cgroup's cpuset.cpus.effective, or -errno. + * + * A sparse cpulist can exceed a page on large systems. cg_read() does a single + * bounded read, so size the buffer for the worst case of NR_CPUS=8192 and + * reject a read that fills it rather than parsing a truncated list. + */ +static int effective_nr_cpus(const char *cgroup, u32 *first, u32 *last) +{ + static char buf[65536]; + + if (cg_read(cgroup, "cpuset.cpus.effective", buf, sizeof(buf))) + return -EIO; + if (strlen(buf) >= sizeof(buf) - 1) + return -EOVERFLOW; + return parse_cpulist(buf, first, last); +} + +/* Run the SYSCALL program to sample scx_bpf_cgroup_nr_cpus() for @path. */ +static int kfunc_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, s64 *nr_cpus) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + u64 cgid; + int err; + + cgid = cgroup_id(path); + if (!cgid) { + SCX_ERR("Failed to read cgroup ID of %s", path); + return -ENOENT; + } + + ctx->skel->bss->query_cgid = cgid; + err = bpf_prog_test_run_opts(bpf_program__fd(ctx->skel->progs.cgroup_nr_cpus_read), + &topts); + if (err || topts.retval) { + SCX_ERR("BPF_PROG_RUN failed for %s (err=%d retval=%d)", + path, err, (int)topts.retval); + return err ?: -EIO; + } + + *nr_cpus = ctx->skel->data->query_nr_cpus; + return 0; +} + +static bool check_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, int expected, + const char *what) +{ + s64 nr_cpus; + + if (kfunc_nr_cpus(ctx, path, &nr_cpus)) + return false; + if (nr_cpus != expected) { + SCX_ERR("%s: expected %d CPUs, got %lld", what, expected, + (long long)nr_cpus); + return false; + } + return true; +} + +/* + * Like check_nr_cpus() but tolerate a transient mismatch. A cpuset css being + * disabled stays attached to its cgroup until it's asynchronously offlined, and + * cpuset_num_cpus() keeps reporting its stale mask until then. + */ +static bool wait_nr_cpus(struct cgroup_nr_cpus_ctx *ctx, const char *path, int expected, + const char *what) +{ + s64 nr_cpus = -1; + int i; + + for (i = 0; i < 1000; i++) { + if (kfunc_nr_cpus(ctx, path, &nr_cpus)) + return false; + if (nr_cpus == expected) + return true; + usleep(1000); + } + SCX_ERR("%s: expected %d CPUs, got %lld", what, expected, (long long)nr_cpus); + return false; +} + +static bool controller_enabled(const char *cgroup, const char *file, const char *controller) +{ + char buf[4096], *saveptr, *token; + + if (cg_read(cgroup, file, buf, sizeof(buf))) + return false; + for (token = strtok_r(buf, "\n ", &saveptr); token; + token = strtok_r(NULL, "\n ", &saveptr)) + if (!strcmp(token, controller)) + return true; + return false; +} + +static void cleanup_ctx(struct cgroup_nr_cpus_ctx *ctx) +{ + bpf_link__destroy(ctx->link); + cgroup_nr_cpus__destroy(ctx->skel); + if (ctx->leaf_created) + cg_destroy(ctx->leaf); + if (ctx->child_created) + cg_destroy(ctx->child); + if (ctx->parent_created) + cg_destroy(ctx->parent); +} + +/* + * Enable @controller in the root's subtree_control if needed. Like the cgroup + * selftests, leave it enabled afterwards: the root is shared, and another + * manager may start relying on the controller while the test runs. + */ +static enum scx_test_status enable_controller(const char *root, const char *controller) +{ + char value[32]; + + if (controller_enabled(root, "cgroup.subtree_control", controller)) + return SCX_TEST_PASS; + if (!controller_enabled(root, "cgroup.controllers", controller)) + return SCX_TEST_SKIP; + + snprintf(value, sizeof(value), "+%s", controller); + if (cg_write(root, "cgroup.subtree_control", value)) + return SCX_TEST_SKIP; + return SCX_TEST_PASS; +} + +static enum scx_test_status setup_cgroups(struct cgroup_nr_cpus_ctx *ctx) +{ + enum scx_test_status status; + char name[64]; + + if (cg_find_unified_root(ctx->root, sizeof(ctx->root), NULL)) + return SCX_TEST_SKIP; + + status = enable_controller(ctx->root, "cpu"); + if (status != SCX_TEST_PASS) + return status; + status = enable_controller(ctx->root, "cpuset"); + if (status != SCX_TEST_PASS) + return status; + + snprintf(name, sizeof(name), "scx_nr_cpus_%d", getpid()); + if (join_path(ctx->parent, sizeof(ctx->parent), ctx->root, name) || + join_path(ctx->child, sizeof(ctx->child), ctx->parent, "child") || + join_path(ctx->leaf, sizeof(ctx->leaf), ctx->child, "leaf")) { + SCX_ERR("Cgroup path is too long"); + return SCX_TEST_FAIL; + } + + if (cg_create(ctx->parent)) { + SCX_ERR("Failed to create cgroup %s", ctx->parent); + return SCX_TEST_FAIL; + } + ctx->parent_created = true; + if (cg_write(ctx->parent, "cgroup.subtree_control", "+cpu +cpuset")) { + SCX_ERR("Failed to enable controllers in %s", ctx->parent); + return SCX_TEST_FAIL; + } + if (cg_create(ctx->child)) { + SCX_ERR("Failed to create cgroup %s", ctx->child); + return SCX_TEST_FAIL; + } + ctx->child_created = true; + if (cg_write(ctx->child, "cgroup.subtree_control", "+cpu")) { + SCX_ERR("Failed to enable cpu in %s", ctx->child); + return SCX_TEST_FAIL; + } + if (cg_create(ctx->leaf)) { + SCX_ERR("Failed to create cgroup %s", ctx->leaf); + return SCX_TEST_FAIL; + } + ctx->leaf_created = true; + + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *arg) +{ + struct cgroup_nr_cpus_ctx ctx = {}; + enum scx_test_status status; + char value[32]; + u32 first, last; + int nr_root, nr_child; + + (void)arg; + + /* + * SCX_ENUM_INIT() exits the process if vmlinux BTF can't be loaded, so + * run it before creating any cgroups that would then be left behind. + */ + ctx.skel = cgroup_nr_cpus__open(); + if (!ctx.skel) { + SCX_ERR("Failed to open skel"); + return SCX_TEST_FAIL; + } + SCX_ENUM_INIT(ctx.skel); + + status = setup_cgroups(&ctx); + if (status != SCX_TEST_PASS) + goto out; + status = SCX_TEST_FAIL; + + nr_root = effective_nr_cpus(ctx.root, NULL, NULL); + nr_child = effective_nr_cpus(ctx.child, &first, &last); + if (nr_root < 0 || nr_child < 0) { + SCX_ERR("Failed to read effective cpusets"); + goto out; + } + /* The effective cpuset can be empty, e.g. under a partition root. */ + if (nr_child < 2) { + status = SCX_TEST_SKIP; + goto out; + } + + ctx.skel->bss->init_cgid = cgroup_id(ctx.leaf); + if (!ctx.skel->bss->init_cgid) { + SCX_ERR("Failed to read cgroup ID of %s", ctx.leaf); + goto out; + } + if (cgroup_nr_cpus__load(ctx.skel)) { + SCX_ERR("Failed to load skel"); + goto out; + } + + /* The kfunc must be callable without a scheduler attached. */ + if (!check_nr_cpus(&ctx, ctx.root, nr_root, "root") || + !check_nr_cpus(&ctx, ctx.child, nr_child, "child") || + !check_nr_cpus(&ctx, ctx.leaf, nr_child, "inherited leaf")) + goto out; + + /* ops.cgroup_init() runs for existing cgroups when attaching. */ + ctx.link = bpf_map__attach_struct_ops(ctx.skel->maps.cgroup_nr_cpus_ops); + if (!ctx.link) { + SCX_ERR("Failed to attach scheduler"); + goto out; + } + if (ctx.skel->data->init_nr_cpus != nr_child) { + SCX_ERR("ops.cgroup_init(): expected %d CPUs, got %lld", nr_child, + (long long)ctx.skel->data->init_nr_cpus); + goto out; + } + + /* Non-contiguous cpuset, observed by the owner and by the inheritor. */ + if (nr_child > 2) { + snprintf(value, sizeof(value), "%u,%u", first, last); + if (cg_write(ctx.child, "cpuset.cpus", value)) { + SCX_ERR("Failed to set cpuset.cpus=%s for %s", value, ctx.child); + goto out; + } + if (!check_nr_cpus(&ctx, ctx.child, 2, "sparse child") || + !check_nr_cpus(&ctx, ctx.leaf, 2, "sparse inherited leaf")) + goto out; + } + + snprintf(value, sizeof(value), "%u", first); + if (cg_write(ctx.child, "cpuset.cpus", value)) { + SCX_ERR("Failed to set cpuset.cpus=%s for %s", value, ctx.child); + goto out; + } + if (!check_nr_cpus(&ctx, ctx.child, 1, "single-CPU child") || + !check_nr_cpus(&ctx, ctx.leaf, 1, "single-CPU inherited leaf")) + goto out; + + /* + * Disabling cpuset below @parent makes @child and @leaf inherit + * @parent's effective cpuset, which spans all of the root's CPUs. + */ + if (cg_write(ctx.parent, "cgroup.subtree_control", "-cpuset")) { + SCX_ERR("Failed to disable cpuset in %s", ctx.parent); + goto out; + } + nr_child = effective_nr_cpus(ctx.parent, NULL, NULL); + if (nr_child < 0) { + SCX_ERR("Failed to read effective cpuset of %s", ctx.parent); + goto out; + } + if (!wait_nr_cpus(&ctx, ctx.child, nr_child, "child after cpuset disable") || + !wait_nr_cpus(&ctx, ctx.leaf, nr_child, "leaf after cpuset disable")) + goto out; + + if (ctx.skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + SCX_ERR("Scheduler exited unexpectedly"); + goto out; + } + + status = SCX_TEST_PASS; +out: + cleanup_ctx(&ctx); + return status; +} + +struct scx_test cgroup_nr_cpus = { + .name = "cgroup_nr_cpus", + .description = "Verify scx_bpf_cgroup_nr_cpus() reports effective cpuset CPU counts", + .run = run, +}; +REGISTER_SCX_TEST(&cgroup_nr_cpus) diff --git a/tools/testing/selftests/sched_ext/config b/tools/testing/selftests/sched_ext/config index affa3cf33470..8173f9170ffe 100644 --- a/tools/testing/selftests/sched_ext/config +++ b/tools/testing/selftests/sched_ext/config @@ -1,6 +1,7 @@ CONFIG_SCHED_CLASS_EXT=y CONFIG_CGROUPS=y CONFIG_CGROUP_SCHED=y +CONFIG_CPUSETS=y CONFIG_EXT_GROUP_SCHED=y CONFIG_BPF=y CONFIG_BPF_SYSCALL=y From a546831dff38b624240a90a6ff6d2b43d9226d51 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 6 Oct 2026 10:29:05 -1000 Subject: [PATCH 47/69] sched_ext: Clear a sub-scheduler's caps before ops.sub_detach() A sub-scheduler can leave standing state behind on the cids delegated to it, cpuperf targets being the obvious case, and the kernel does not track who wrote it. Restoring it is the parent's job and ops.sub_detach() is the place for it. Today the parent can't rely on that: the dying sub-scheduler keeps every cap until it is marked dead after ops.exit(), so its ops.exit() or a late timer can write after the parent has cleaned up. Clear every cap of the sub-scheduler before ops.sub_detach(). The pshard caps go first as a pending sync recomputes ecaps from them, and ecaps are then zeroed directly. A grant racing the clear is refused under the child's pshard lock, so nothing hands the caps back. Fixes: 86094b95efcf ("sched_ext: Add per-shard cap delegation for sub-schedulers") Signed-off-by: Tejun Heo --- kernel/sched/ext/internal.h | 7 ++++++ kernel/sched/ext/sub.c | 49 ++++++++++++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index d65fec631bdf..b5dc3c159d45 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -864,6 +864,10 @@ struct sched_ext_ops { /** * @sub_detach: Detach a sub-scheduler * @args: argument container, see the struct definition + * + * The sub-scheduler holds no caps by this point and can no longer + * affect any cid. Whatever was delegated to it, e.g. cpuperf targets, + * is the parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -947,6 +951,9 @@ struct sched_ext_ops { * ops.exit() is also called on ops.init() failure, which is a bit * unusual. This is to allow rich reporting through @info on how * ops.init() failed. + * + * A sub-scheduler holds no caps by the time ops.exit() runs. Its + * cap-gated kfuncs are denied. */ void (*exit)(struct scx_exit_info *info); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 48e17aeb1cb6..0d2fb7387cd3 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1104,6 +1104,34 @@ void scx_offline_ecaps(struct rq *rq) } } +/* + * Clear every cap @sch holds. The pshard caps go first as they are the source a + * pending sync recomputes ecaps from. ecaps are then zeroed directly for the + * cap checks. + */ +static void clear_all_caps(struct scx_sched *sch) +{ + s32 si, cpu; + u32 cap_bit; + + /* enable may have failed before scx_alloc_pshards() */ + if (!sch->pshard) + return; + + for (si = 0; si < sch->nr_pshards; si++) { + struct scx_pshard *ps = sch->pshard[si]; + + guard(raw_spinlock_irqsave)(&ps->lock); + for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++) + scx_cmask_clear(&ps->caps[cap_bit].cmask); + } + + for_each_possible_cpu(cpu) { + guard(rq_lock_irqsave)(cpu_rq(cpu)); + WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0); + } +} + /* * @pcpu's sched was unhashed before the grace period, so nothing re-queues its * sync node. Remove the node from @rq's pending list so the pcpu can be freed. @@ -1583,6 +1611,12 @@ void scx_sub_disable(struct scx_sched *sch) scx_unlink_sched(sch); + /* + * The parent restores what it delegated in ops.sub_detach(). @sch must + * hold no caps by then. + */ + clear_all_caps(sch); + mutex_unlock(&scx_enable_mutex); /* @@ -2266,8 +2300,8 @@ static s32 sub_cap_preamble(u64 cgroup_id, u64 caps, const struct bpf_prog_aux * * All-or-nothing keeps the caller-visible result binary per cid, so the denied * mask is one mask to interpret rather than a per-cap matrix. * - * Return 0 on full success, -EPERM if any cid was refused, or a negative - * errno on other failures. + * Return 0 on full success, -EPERM if any cid was refused, -ENODEV if the child + * is gone or being disabled, or a negative errno on other failures. */ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, const struct scx_cmask *cmask__arena, @@ -2321,6 +2355,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, scoped_guard (raw_spinlock, &pps->lock) { guard(raw_spinlock_nested)(&cps->lock); + /* the child is being disabled, see clear_all_caps() */ + if (unlikely(READ_ONCE(child->aborting))) { + ret = -ENODEV; + goto out; + } + /* * Narrow granted_cids to cids the parent holds every * requested cap on. All-or-nothing per cid. @@ -2373,9 +2413,10 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, } } + ret = any_denied ? -EPERM : 0; +out: caps_updated_deliver(&to_deliver); - - return any_denied ? -EPERM : 0; + return ret; } /** From 457fff158adcd2a60f2a908957e193cb78649dbf Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Oct 2026 07:52:12 -1000 Subject: [PATCH 48/69] sched_ext: Add ops.sub_cid_sched_updated() to report the sched running on a cid A parent sched that delegates cpus to sub-scheds has no way of telling how much each delegatee is using them. A parent that sizes its delegations by demand cannot measure it, and one that wants to hand a cid from one sub to another cannot see which sub it would take the cid from. Add ops.sub_cid_sched_updated(), invoked on a sched when what it sees running on a cid changes: NONE when no task of its subtree runs there, SELF for its own task, or a direct child's cgroup id. Changes inside a child's subtree are not reported. The op costs one pointer compare per context switch when the sched does not change, a notification reaches only the scheds that see a change, and the whole mechanism sits behind a static key that is on only while a loaded sched implements the op. After a sched's tasks are gone, its disable closes every rq session still pointing at it, so nothing refers to it once it is freed. The op is cid-form only like the other sub ops and runs with the rq lock held on the context switch path. Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 20 ++++- kernel/sched/ext/internal.h | 29 +++++++ kernel/sched/ext/sub.c | 166 ++++++++++++++++++++++++++++++++++++ kernel/sched/ext/sub.h | 6 ++ kernel/sched/sched.h | 2 + 5 files changed, 222 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 96c904b39601..248b39d09ce3 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3333,6 +3333,8 @@ static void scx_start_task_running(struct rq *rq, struct task_struct *p) if (p->scx.flags & SCX_TASK_RUN_TRACKED) return; + scx_cid_sched_update(rq, sch); + if (SCX_HAS_OP(sch, running)) SCX_CALL_OP_TASK(sch, running, rq, p); @@ -3593,8 +3595,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, } switch_class: - if (next && next->sched_class != &ext_sched_class) + if (next && next->sched_class != &ext_sched_class) { + scx_cid_sched_update(rq, NULL); switch_class(rq, next); + } } static void kick_sync_wait_bal_cb(struct rq *rq) @@ -4724,6 +4728,13 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p) static void switched_from_scx(struct rq *rq, struct task_struct *p) { + /* + * A class change of the running task: sched_change_begin() put @p with + * no successor, which leaves rq->scx.sched set. + */ + if (task_current_donor(rq, p)) + scx_cid_sched_update(rq, NULL); + if (task_dead_and_done(p)) return; @@ -7047,6 +7058,8 @@ static void scx_root_disable(struct scx_sched *sch) } } + scx_ops_cid_sched_updated_disable(sch); + /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); static_branch_disable(&__scx_is_cid_type); @@ -8255,6 +8268,8 @@ static void scx_root_enable_workfn(struct kthread_work *work) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); + scx_ops_cid_sched_updated_enable(sch); + if (sch->ops.cpu_acquire || sch->ops.cpu_release) sch->ops.flags |= SCX_OPS_HAS_CPU_PREEMPT; @@ -8905,6 +8920,7 @@ static void sched_ext_ops_cid__set_cmask(struct task_struct *p, const struct scx static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_args *args) {} static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} +static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -8939,6 +8955,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .sub_detach = sched_ext_ops__sub_detach, .sub_caps_updated = sched_ext_ops__sub_caps_updated, .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, + .sub_cid_sched_updated = sched_ext_ops__sub_cid_sched_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init_cids = sched_ext_ops__init_cids, @@ -11678,6 +11695,7 @@ static int __init scx_init(void) CID_OFFSET_MATCH(sub_detach, sub_detach); CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); + CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated); CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b5dc3c159d45..8dcab02a38ab 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -366,6 +366,18 @@ struct scx_sub_detach_args { char *cgroup_path; }; +/* + * What a sched sees running on a cid, as reported by + * ops.sub_cid_sched_updated(): NONE when the cid is idle or runs a task outside + * the sched's subtree, SELF for the sched itself, or a direct child by its + * cgroup id. The sentinels lie outside the cgroup id space, whose low 32 bits + * are an idr value of at most INT_MAX. + */ +enum scx_cid_sched_consts { + SCX_CID_SCHED_NONE = (u64)-2, + SCX_CID_SCHED_SELF = (u64)-1, +}; + /** * struct sched_ext_ops - Operation table for BPF scheduler implementation * @@ -904,6 +916,22 @@ struct sched_ext_ops { */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + /** + * @sub_cid_sched_updated: The sched running on a cid changed + * @cid: cid whose running sched changed + * @sched: what this sched now sees running there + * + * Invoked when the sched owning the task running on @cid changes. + * @sched is %SCX_CID_SCHED_NONE when no task of this sched's subtree + * runs there, %SCX_CID_SCHED_SELF for its own task, or the cgroup id of + * the direct child whose subtree the task belongs to. Changes inside a + * child's subtree, whether across tasks or nested sub-scheds, are not + * reported. Every sched for which @sched changes is notified. + * + * Runs with the rq lock held on the context switch path. + */ + void (*sub_cid_sched_updated)(s32 cid, u64 sched); + /* * All online ops must come before ops.cpu_online(). */ @@ -1161,6 +1189,7 @@ struct sched_ext_ops_cid { void (*sub_detach)(struct scx_sub_detach_args *args); void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps); void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); + void (*sub_cid_sched_updated)(s32 cid, u64 sched); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); s32 (*init_cids)(void); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 0d2fb7387cd3..51a53467cf98 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -28,6 +28,13 @@ */ DEFINE_STATIC_KEY_FALSE(__scx_has_subs); +/* + * On while a loaded sched implements ops.sub_cid_sched_updated(), see + * scx_ops_cid_sched_updated_enable(). + */ +static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled); +static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */ + /* latched at root enable before any rescue runs */ static s32 scx_rescue_bw_1024; static s64 scx_rescue_quantum_ns; @@ -92,6 +99,161 @@ struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sche return scx_skip_subtree_pre(pos, root); } +/** + * scx_cid_sched_id_from - Resolve what @from sees when @sch runs on a cid + * @sch: sched whose task runs there, %NULL for none + * @from: sched asking + * + * Return %SCX_CID_SCHED_NONE when @sch is %NULL or outside @from's subtree, + * %SCX_CID_SCHED_SELF for @from itself, or else the cgroup id of the direct + * child of @from on the way down to @sch. + */ +static u64 scx_cid_sched_id_from(struct scx_sched *sch, struct scx_sched *from) +{ + if (!sch || !scx_is_descendant(sch, from)) + return SCX_CID_SCHED_NONE; + if (sch == from) + return SCX_CID_SCHED_SELF; + return sch->ancestors[from->level + 1]->ops.sub_cgroup_id; +} + +/* notify @from of what it now sees running on @rq's cid */ +static void scx_cid_notify_sched_updated(struct rq *rq, struct scx_sched *sch, + struct scx_sched *from) +{ + if (SCX_HAS_OP(from, sub_cid_sched_updated)) + SCX_CALL_OP(from, sub_cid_sched_updated, rq, __scx_cpu_to_cid(cpu_of(rq)), + scx_cid_sched_id_from(sch, from)); +} + +/** + * scx_cid_sched_update - Update rq->scx.sched and notify the scheds affected + * @rq: rq whose running session opens or closes + * @sch: sched starting a session, %NULL to close the open one + * + * Every sched whose scx_cid_sched_id_from() value changes is notified of the + * new one. Above the deepest common ancestor of the two scheds, both map to the + * same child, so the walk stops there. With root R, children A and B, and A's + * child A1, a switch from an A1 task to a B task reports: + * + * R R: A -> B + * / \ + * A B A: A1 -> NONE B: NONE -> SELF + * | + * A1 A1: SELF -> NONE + * + * When the task on one side is not an ext task, there is no common ancestor and + * only the other side is walked. + */ +void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) +{ + struct scx_sched *prev = rq->scx.sched; + s32 level, common = -1; + + lockdep_assert_rq_held(rq); + + if (!static_branch_unlikely(&__scx_ops_cid_sched_updated_enabled) || prev == sch) + return; + + rq->scx.sched = sch; + + if (prev && sch) { + for (level = min(prev->level, sch->level); level >= 0; level--) + if (prev->ancestors[level] == sch->ancestors[level]) + break; + common = level; + scx_cid_notify_sched_updated(rq, sch, prev->ancestors[common]); + } + if (prev) + for (level = common + 1; level <= prev->level; level++) + scx_cid_notify_sched_updated(rq, sch, prev->ancestors[level]); + if (sch) + for (level = common + 1; level <= sch->level; level++) + scx_cid_notify_sched_updated(rq, sch, sch->ancestors[level]); +} + +/** + * scx_ops_cid_sched_updated_enable - Maintain rq->scx.sched if @sch has the op + * @sch: sched being enabled, its has_op filled in + * + * rq->scx.sched is maintained only while a loaded sched implements + * ops.sub_cid_sched_updated(), so that everyone else pays one static branch per + * session open or close. The first such sched turns the key on and then + * initializes every rq from the task running there, in that order so that no + * session change in between goes unrecorded. While the key is off every + * rq->scx.sched is NULL, see scx_ops_cid_sched_updated_disable(). + */ +void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) +{ + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + if (!SCX_HAS_OP(sch, sub_cid_sched_updated) || scx_nr_ops_cid_sched_updated++) + return; + + static_branch_enable(&__scx_ops_cid_sched_updated_enabled); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + struct task_struct *p; + + guard(rq_lock_irqsave)(rq); + p = rq->donor; + if (p->sched_class == &ext_sched_class && + (p->scx.flags & SCX_TASK_RUN_TRACKED)) + rq->scx.sched = scx_task_sched(p); + } +} + +/** + * scx_ops_cid_sched_updated_disable - Stop maintaining rq->scx.sched for @sch + * @sch: sched being disabled, its tasks all gone + * + * Close the session on every rq that still points at @sch, before ops.exit(), + * so that no sched is notified of @sch once it is gone. The last sched with the + * op turns the key off first and clears every rq, so while the key is off every + * rq->scx.sched is NULL. A sched whose enable failed before its has_op fill was + * never counted, so has_op gates the decrement. + */ +void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) +{ + bool last; + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + last = SCX_HAS_OP(sch, sub_cid_sched_updated) && !--scx_nr_ops_cid_sched_updated; + if (last) + static_branch_disable(&__scx_ops_cid_sched_updated_enabled); + + /* + * Re-homing restarts a running task's session under its new sched. A + * task that blocked while its cpu's dispatch had the rq lock dropped is + * put and set next again without a restart, so rq->scx.sched keeps @sch + * until the pick completes: + * + * cpu 0, __schedule() cpu 1, scx_sub_disable(@sch) + * T of @sch blocks, stops running + * dispatch drops the rq lock + * re-home T, nothing to restart + * this sweep + * pick N, N's session opens + * + * Closing the session here reports NONE to @sch and its ancestors now + * and lets the pick report N's sched from NONE instead of from @sch. + */ + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + if (rq->scx.sched == sch) + scx_cid_sched_update(rq, NULL); + if (last) + rq->scx.sched = NULL; + } +} + static struct scx_sched *scx_find_sub_sched(u64 cgroup_id) { return rhashtable_lookup(&scx_sched_hash, &cgroup_id, @@ -1599,6 +1761,8 @@ void scx_sub_disable(struct scx_sched *sch) scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); + scx_ops_cid_sched_updated_disable(sch); + /* * All tasks are moved off of @sch but there may still be on-going * operations (e.g. ops.select_cpu()). Drain them by flushing RCU. Use @@ -1827,6 +1991,8 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); + scx_ops_cid_sched_updated_enable(sch); + percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index c0174e523f2a..3cd31b24e649 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -16,6 +16,9 @@ struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); +void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch); +void scx_ops_cid_sched_updated_enable(struct scx_sched *sch); +void scx_ops_cid_sched_updated_disable(struct scx_sched *sch); void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch); struct cgroup *sch_cgroup(struct scx_sched *sch); void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch); @@ -94,6 +97,9 @@ static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; } static inline struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root) { return NULL; } +static inline void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch) {} +static inline void scx_ops_cid_sched_updated_enable(struct scx_sched *sch) {} +static inline void scx_ops_cid_sched_updated_disable(struct scx_sched *sch) {} static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {} static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; } static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0a34f0da1ec3..fc65296f5c74 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -833,6 +833,8 @@ struct scx_rq { #endif u64 clock; /* current per-rq clock -- see scx_bpf_now() */ #ifdef CONFIG_EXT_SUB_SCHED + /* sched of the open running session, see scx_cid_sched_update() */ + struct scx_sched *sched; struct llist_head ecaps_to_sync; /* pending ecaps syncs */ struct task_struct *sub_dispatch_prev; #endif From 9a206b2c7bd2e4b3e0deb04740b044d96c1d66b3 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Oct 2026 07:52:12 -1000 Subject: [PATCH 49/69] sched_ext: scx_qmap: Size the cid range buffer for large machines The hier rows' cid range buffer holds 128 characters. When no two consecutive cids share an owner, which SMT siblings numbered far apart produce, every cid prints on its own and the list is cut off with "...". Size the buffer for that worst case at the build's cpu cap, off the stack. Signed-off-by: Tejun Heo --- tools/sched_ext/scx_qmap.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index d5226e071657..96e485c9e877 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -162,7 +162,9 @@ static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_ /* partition summary + one row per sched: weight, cpus, dispatch rate, cids */ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid) { - char ranges[128], who[16]; + /* worst case, no ranges: a comma and up to six digits per cid */ + static char ranges[SCX_QMAP_MAX_CPUS * 7]; + char who[16]; const char *rr = "-"; double secs; u32 i; From 3d7c2f550eefe30e90fbb031897b0170d8dee303 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 5 Oct 2026 07:52:12 -1000 Subject: [PATCH 50/69] sched_ext: scx_qmap: Show actual cid use per participant qmap's hier stats show how much cid-time the partition handed each participant, not how much of it was used. Track the sched running on each cid from ops.sub_cid_sched_updated() and charge the intervals to the participant, then print the use next to the allocation. Signed-off-by: Tejun Heo --- tools/sched_ext/scx_qmap.bpf.c | 87 ++++++++++++++++++++++++++++++++-- tools/sched_ext/scx_qmap.c | 38 ++++++++++++--- tools/sched_ext/scx_qmap.h | 11 +++++ 3 files changed, 127 insertions(+), 9 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 3566e3e02e21..7f69394556b6 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1782,13 +1782,90 @@ static void redistribute(void) } /* - * Userspace pokes this (PROG_RUN) to bring alloc_ns[] current before reading - * it for the stats display. Skipping when the partition guard is held is - * fine - alloc_ts is untouched, so the elapsed time is charged next time. + * Owner id for a @sched value of ops.sub_cid_sched_updated(). A child is + * attached before its first task runs and its tasks are re-homed before it + * detaches, so a child's cgroup id always has its slot. + */ +static s32 cid_sched_owner(u64 sched) +{ + s32 i; + + if (sched == SCX_CID_SCHED_NONE) + return CID_NONE; + if (sched == SCX_CID_SCHED_SELF) + return CID_SELF; + bpf_for(i, 0, MAX_SUB_SCHEDS) + if (qa.sub_sched_ctxs[i].cgroup_id == sched) + return i; + return CID_NONE; +} + +/* used_ns[] is summed from every cpu, hence the atomic adds */ +static void cid_sched_charge(s32 cid, u64 now) +{ + s32 owner = qa.cid_sched[cid]; + u64 delta = now - qa.cid_sched_since[cid]; + + if (owner >= 0 && owner < MAX_SUB_SCHEDS) + __sync_fetch_and_add(&qa.used_ns[owner], delta); + else if (owner == CID_SELF) + __sync_fetch_and_add(&qa.self_used_ns, delta); + qa.cid_sched_since[cid] = now; +} + +void BPF_STRUCT_OPS(qmap_sub_cid_sched_updated, s32 cid, u64 sched) +{ + if (cid < 0 || cid >= SCX_QMAP_MAX_CPUS) + return; + + cid_sched_charge(cid, bpf_ktime_get_ns()); + qa.cid_sched[cid] = cid_sched_owner(sched); +} + +/* + * Snapshot the used time for the stats display: the closed intervals plus the + * ones still open. The reads race the notifications on other cpus, so an + * interval closing in between can be missing from one snapshot or counted in + * two. The next snapshot evens it out and the display floors a negative + * difference at zero. + */ +static void snapshot_used(void) +{ + u64 now = bpf_ktime_get_ns(); + s32 nr_cids = qa.nr_cids; + s32 cid, i; + + if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS) + return; + + bpf_for(i, 0, MAX_SUB_SCHEDS) + qa.used_snap_ns[i] = qa.used_ns[i]; + qa.self_used_snap_ns = qa.self_used_ns; + + bpf_for(cid, 0, nr_cids) { + s32 owner = qa.cid_sched[cid]; + u64 since = qa.cid_sched_since[cid]; + + /* restarted after @now by a notification on another cpu */ + if (since > now) + continue; + if (owner >= 0 && owner < MAX_SUB_SCHEDS) + qa.used_snap_ns[owner] += now - since; + else if (owner == CID_SELF) + qa.self_used_snap_ns += now - since; + } +} + +/* + * Userspace pokes this (PROG_RUN) to bring alloc_ns[] and the used snapshot + * current before reading them for the stats display. Skipping the alloc part + * when the partition guard is held is fine - alloc_ts is untouched, so the + * elapsed time is charged next time. */ SEC("syscall") int flush_alloc(void *ctx) { + snapshot_used(); if (part_try_start()) { account_alloc(); part_end(); @@ -1940,6 +2017,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init) /* cache the cid count, trusted to be <= SCX_QMAP_MAX_CPUS hereafter */ qa.nr_cids = nr_cids; + bpf_for(i, 0, nr_cids) + qa.cid_sched[i] = CID_NONE; + /* cmasks are embedded in qa, so they only need initializing */ cmask_init(&qa.idle_cids.mask, 0, nr_cids); cmask_init(&qa.rr_cids.mask, 0, nr_cids); @@ -2151,6 +2231,7 @@ SCX_OPS_CID_DEFINE(qmap_ops, .sub_detach = (void *)qmap_sub_detach, .sub_caps_updated = (void *)qmap_sub_caps_updated, .sub_ecaps_updated = (void *)qmap_sub_ecaps_updated, + .sub_cid_sched_updated = (void *)qmap_sub_cid_sched_updated, .init_cids = (void *)qmap_init_cids, .init = (void *)qmap_init, .exit = (void *)qmap_exit, diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index 96e485c9e877..e83d5e2147dd 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -105,6 +105,8 @@ struct hier_prev { u64 alloc_ns[MAX_SUB_SCHEDS]; u64 self_alloc_ns; u64 alloc_window_ns; + u64 used_snap_ns[MAX_SUB_SCHEDS]; + u64 self_used_snap_ns; u64 nr_dsps[MAX_SUB_SCHEDS]; u64 nr_reenq_cap; u64 nr_reenq_immed; @@ -159,6 +161,18 @@ static void format_cid_ranges(struct qmap_arena *qa, s32 owner, char *buf, size_ strcpy(buf, "-"); } +/* + * Delta of a cumulative ns counter over the interval, as a fraction of the + * interval. The used snapshot can briefly run behind the previous one (see + * snapshot_used()), hence the floor. + */ +static double delta_ratio(u64 cur, u64 prev, double secs) +{ + s64 delta = cur - prev; + + return secs > 0 && delta > 0 ? delta / (secs * 1e9) : 0.0; +} + /* partition summary + one row per sched: weight, cpus, dispatch rate, cids */ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cgid) { @@ -207,15 +221,24 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg prev->nr_inject_attempts = qa->nr_inject_attempts; prev->nr_rescue_dsp = qa->nr_rescue_dsp; - printf("hier : %-4s %10s %4s %6s %8s %s\n", - "", "cgroup", "w", "alloc", "disp/s", "cids"); + /* + * alloc is the cid-time the partition handed each participant, and used + * is the cid-time its tasks actually ran, per + * ops.sub_cid_sched_updated(). Both are in cpus over the window. + */ + printf("hier : %-4s %10s %4s %6s %6s %8s %s\n", + "", "cgroup", "w", "alloc", "used", "disp/s", "cids"); format_cid_ranges(qa, CID_SELF, ranges, sizeof(ranges)); - printf("hier : %-4s %10llu %4u %6.2f %8s %s\n", "self", + printf("hier : %-4s %10llu %4u %6.2f %6.2f %8s %s\n", "self", (unsigned long long)own_cgid, 100, - secs > 0 ? (qa->self_alloc_ns - prev->self_alloc_ns) / (secs * 1e9) : 0.0, + delta_ratio(qa->self_alloc_ns, prev->self_alloc_ns, secs), + delta_ratio(qa->self_used_snap_ns, prev->self_used_snap_ns, secs), "-", ranges); prev->self_alloc_ns = qa->self_alloc_ns; + /* used accrues without a window, so prev moves only with the window */ + if (secs > 0) + prev->self_used_snap_ns = qa->self_used_snap_ns; for (i = 0; i < MAX_SUB_SCHEDS; i++) { struct sub_sched_ctx *sc = &qa->sub_sched_ctxs[i]; @@ -225,12 +248,15 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg snprintf(who, sizeof(who), "sub%u", i); format_cid_ranges(qa, i, ranges, sizeof(ranges)); - printf("hier : %-4s %10llu %4u %6.2f %8.1f %s\n", who, + printf("hier : %-4s %10llu %4u %6.2f %6.2f %8.1f %s\n", who, (unsigned long long)sc->cgroup_id, sc->weight, - secs > 0 ? (qa->alloc_ns[i] - prev->alloc_ns[i]) / (secs * 1e9) : 0.0, + delta_ratio(qa->alloc_ns[i], prev->alloc_ns[i], secs), + delta_ratio(qa->used_snap_ns[i], prev->used_snap_ns[i], secs), secs > 0 ? (sc->nr_dsps - prev->nr_dsps[i]) / secs : 0.0, ranges); prev->alloc_ns[i] = qa->alloc_ns[i]; + if (secs > 0) + prev->used_snap_ns[i] = qa->used_snap_ns[i]; prev->nr_dsps[i] = sc->nr_dsps; } } diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index 089c5176c4ec..949459d06a18 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -163,6 +163,17 @@ struct qmap_arena { u64 alloc_ts; /* last accounting timestamp */ u64 alloc_window_ns; /* total accounted time, the alloc denominator */ + /* + * The per-cid fields are written only by that cid's notifications and + * read by flush_alloc() for the snapshot userspace displays. + */ + s32 cid_sched[SCX_QMAP_MAX_CPUS]; /* per cid: owner id of the sched running there */ + u64 cid_sched_since[SCX_QMAP_MAX_CPUS]; /* when that last changed */ + u64 used_ns[MAX_SUB_SCHEDS]; /* per child slot, closed intervals */ + u64 self_used_ns; + u64 used_snap_ns[MAX_SUB_SCHEDS]; /* used_ns[] plus the intervals open at the flush */ + u64 self_used_snap_ns; + /* bpf-internal cmasks (embedded, see struct qmap_cmask) */ struct qmap_cmask self_cids; /* cids this node runs its own tasks on */ struct qmap_cmask avail_cids; /* cids with caps in effect on the cpu */ From da3bad759ef2df9db38ad851a061d719091d74dd Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 7 Oct 2026 23:15:39 -1000 Subject: [PATCH 51/69] sched_ext: Add ops.sub_child_ecaps_updated() to report a child's effective cap changes A grant or revoke only records the target caps. They take effect on a cid at its next dispatch, and nothing tells the parent when. Without that the parent cannot act on a revoke: a child that ran a cid at a low cpuperf target leaves the target there when PERF is revoked, as the kernel resets targets only at root enable. Nor can the parent tell when it may schedule on the cid again. Add ops.sub_child_ecaps_updated(), delivered to the direct parent right after the child's own ops.sub_ecaps_updated() with the child's cgroup id and the same before and after caps, in the same dispatch context. Both deliveries are suppressed while the child is bypassing and replayed together afterwards. A disabled child reports nothing: ops.sub_detach() is where the parent restores what it had delegated. A sub is now bypassed before it is linked, so that the grants queued for it from ops.sub_attach() are consumed while it is bypassed and replayed to it and the parent once it is enabled. Otherwise a sync consumed before the sub's ops are registered would reach the parent but never the sub. v2: Drop the disable report with its sleep lock and nested-dispatch gate, the parent restores from ops.sub_detach() instead. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 4 +++ kernel/sched/ext/internal.h | 30 +++++++++++++++++-- kernel/sched/ext/sub.c | 59 ++++++++++++++++++++++++++----------- 3 files changed, 72 insertions(+), 21 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 248b39d09ce3..2095b78b6bf6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -8921,6 +8921,7 @@ static void sched_ext_ops_cid__enable(struct task_struct *p, struct scx_enable_a static void sched_ext_ops__sub_caps_updated(const struct scx_cmask *cmask__arena, u64 caps) {} static void sched_ext_ops__sub_ecaps_updated(s32 cid, u64 before, u64 after) {} static void sched_ext_ops__sub_cid_sched_updated(s32 cid, u64 sched) {} +static void sched_ext_ops__sub_child_ecaps_updated(u64 cgroup_id, s32 cid, u64 before, u64 after) {} static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .select_cid = sched_ext_ops__select_cpu, @@ -8956,6 +8957,7 @@ static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = { .sub_caps_updated = sched_ext_ops__sub_caps_updated, .sub_ecaps_updated = sched_ext_ops__sub_ecaps_updated, .sub_cid_sched_updated = sched_ext_ops__sub_cid_sched_updated, + .sub_child_ecaps_updated = sched_ext_ops__sub_child_ecaps_updated, .cid_online = sched_ext_ops__cpu_online, .cid_offline = sched_ext_ops__cpu_offline, .init_cids = sched_ext_ops__init_cids, @@ -11552,6 +11554,7 @@ static const u32 scx_kf_allow_flags[] = { [SCX_OP_IDX(sub_attach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_detach)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(sub_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, + [SCX_OP_IDX(sub_child_ecaps_updated)] = SCX_KF_ALLOW_ENQUEUE | SCX_KF_ALLOW_DISPATCH, [SCX_OP_IDX(cpu_online)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(cpu_offline)] = SCX_KF_ALLOW_UNLOCKED, [SCX_OP_IDX(init_cids)] = SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT_CIDS, @@ -11696,6 +11699,7 @@ static int __init scx_init(void) CID_OFFSET_MATCH(sub_caps_updated, sub_caps_updated); CID_OFFSET_MATCH(sub_ecaps_updated, sub_ecaps_updated); CID_OFFSET_MATCH(sub_cid_sched_updated, sub_cid_sched_updated); + CID_OFFSET_MATCH(sub_child_ecaps_updated, sub_child_ecaps_updated); CID_OFFSET_MATCH(init_cids, init_cids); CID_OFFSET_MATCH(init, init); CID_OFFSET_MATCH(exit, exit); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 8dcab02a38ab..2b637973244c 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -878,8 +878,9 @@ struct sched_ext_ops { * @args: argument container, see the struct definition * * The sub-scheduler holds no caps by this point and can no longer - * affect any cid. Whatever was delegated to it, e.g. cpuperf targets, - * is the parent's to restore here. + * affect any cid. Its caps went away without a + * sub_child_ecaps_updated() report, so whatever was delegated to it, + * e.g. cpuperf targets, is the parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -932,6 +933,25 @@ struct sched_ext_ops { */ void (*sub_cid_sched_updated)(s32 cid, u64 sched); + /** + * @sub_child_ecaps_updated: A child's effective caps on a cid changed + * @cgroup_id: cgroup id of the direct child + * @cid: the cid whose effective caps changed + * @before: the child's effective caps as of the last delivery + * @after: the child's effective caps now + * + * Invoked right after the child's sub_ecaps_updated(), once a grant or + * revoke is in effect on the cpu. From here on the parent can act on a + * cap the child lost, e.g. reset the cpuperf target or schedule on the + * cid. A detaching child's caps go away without a report, see + * sub_detach(). A cpu going offline drops caps without a report. + * + * Runs in dispatch context with rq lock held, and can perform all + * operations allowed in ops.dispatch() including inserting/moving + * tasks. + */ + void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after); + /* * All online ops must come before ops.cpu_online(). */ @@ -1190,6 +1210,7 @@ struct sched_ext_ops_cid { void (*sub_caps_updated)(const struct scx_cmask *cmask__arena, u64 caps); void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); void (*sub_cid_sched_updated)(s32 cid, u64 sched); + void (*sub_child_ecaps_updated)(u64 cgroup_id, s32 cid, u64 before, u64 after); void (*cid_online)(s32 cid); void (*cid_offline)(s32 cid); s32 (*init_cids)(void); @@ -1453,7 +1474,10 @@ struct scx_sched_pcpu { struct llist_node ecaps_to_sync_node; /* owed a forced update_idle() re-notify on this cpu */ bool idle_renotify; - /* effective caps as of the last sub_ecaps_updated() delivery */ + /* + * ecaps as of the last sub_ecaps_updated() and + * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps(). + */ u64 reported_ecaps; /* diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 51a53467cf98..c7c95661280b 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1121,27 +1121,43 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) lost_all |= lost; /* - * Tell the sched its effective caps on this cid changed. The - * invocation is equivalent to the dispatch path and may drop - * and re-acquire the rq lock temporarily while the rest of - * @batch is held privately, see scx_discard_ecaps_to_sync(). - * The dispatch kfuncs resolve their context on the executing - * cpu, which under core scheduling can differ from @rq's cpu, - * so the context is set up there. The rq recorded in it keeps - * the dispatches targeting @rq. + * Tell the sched and its parent that the sched's effective caps + * on this cid changed. The invocations are equivalent to the + * dispatch path and may drop and re-acquire the rq lock + * temporarily while the rest of @batch is held privately, see + * scx_discard_ecaps_to_sync(). The dispatch kfuncs resolve + * their context on the executing cpu, which under core + * scheduling can differ from @rq's cpu, so the context is set + * up there. The rq recorded in it keeps the dispatches + * targeting @rq. + * + * Bypass propagates down the hierarchy, so a sched that isn't + * bypassing has no bypassing parent. The child's bypass state + * gates both deliveries. Both report the same before value, so + * one reported_ecaps covers them. */ - if (ecaps != pcpu->reported_ecaps && - SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && - !scx_bypassing(pcpu->sch, cpu)) { - struct scx_dsp_ctx *dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) { + struct scx_sched *parent = scx_parent(pcpu->sch); + struct scx_dsp_ctx *dspc; - dspc->rq = rq; /* stash @prev so nested dispatches can access it */ rq->scx.sub_dispatch_prev = prev; - SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu), - pcpu->reported_ecaps, ecaps); + if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) { + dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; + dspc->rq = rq; + SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, + scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps); + scx_flush_dispatch_buf(pcpu->sch, rq); + } + if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) { + dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx; + dspc->rq = rq; + SCX_CALL_OP(parent, sub_child_ecaps_updated, rq, + pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu), + pcpu->reported_ecaps, ecaps); + scx_flush_dispatch_buf(parent, rq); + } rq->scx.sub_dispatch_prev = NULL; - scx_flush_dispatch_buf(pcpu->sch, rq); pcpu->reported_ecaps = ecaps; } @@ -1940,6 +1956,15 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; + /* + * Bypass before @sch is linked and grants can reach it. The parent's + * delivery advances reported_ecaps, so a sync consumed before @sch's + * ops are registered would reach the parent and never be replayed to + * @sch. While bypassed, the syncs are consumed without a delivery and + * replayed to both at unbypass. + */ + scx_bypass(sch, true); + ret = scx_link_sched(sch); if (ret) goto err_disable; @@ -1985,8 +2010,6 @@ void scx_sub_enable_workfn(struct kthread_work *work) } sch->sub_attached = true; - scx_bypass(sch, true); - for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); From 0ded98e4c014d8bc9ced2dc7a4d1fb290a594d48 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 9 Oct 2026 10:48:28 -1000 Subject: [PATCH 52/69] sched_ext: scx_qmap: Reset a child's cpuperf targets once its PERF revoke takes effect A child that ran a cid at a low cpuperf target leaves the target there when PERF is revoked, since the kernel resets targets only at root enable. Reset the target from ops.sub_child_ecaps_updated() once the revoke is in effect. A detach reports nothing, so ops.sub_detach() resets the child's cids, and the pool unless another child holds it. Count the deliveries in the hier stats. v2: Reset from ops.sub_detach() for a detaching child's cids. v3: Skip the pool at detach while another child holds it (Andrea Righi). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/scx_qmap.bpf.c | 39 ++++++++++++++++++++++++++++++++-- tools/sched_ext/scx_qmap.c | 7 ++++-- tools/sched_ext/scx_qmap.h | 1 + 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 7f69394556b6..61143a2d79f8 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -2167,12 +2167,36 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args) void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args) { - s32 i; + u64 cgid = args->ops->sub_cgroup_id; + s32 nr_cids = qa.nr_cids; + s32 pos = qa.part.rr_pos; + u64 holder_cgid; + bool reset_pool; + s32 i, cid; + + if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || pos < 0 || pos >= MAX_PARTS) { + scx_bpf_error("-ERANGE"); + return; + } + + /* + * The child's caps are gone without a report, see ops.sub_detach(). + * cpuperf targets persist until the next write. Reset the child's excl + * cids, and the pool unless another child holds it: that child's revoke + * report or its own detach resets it then. + */ + holder_cgid = qa.part.rr_slots[pos]; + reset_pool = !holder_cgid || holder_cgid == cgid; for (i = 0; i < MAX_SUB_SCHEDS; i++) { - if (qa.sub_sched_ctxs[i].cgroup_id != args->ops->sub_cgroup_id) + if (qa.sub_sched_ctxs[i].cgroup_id != cgid) continue; + bpf_arena_for(cid, 0, nr_cids) + if (cmask_test(cid, &qa.sub_sched_ctxs[i].granted_cids.mask) || + (reset_pool && cmask_test(cid, &qa.rr_cids.mask))) + scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE); + qa.sub_sched_ctxs[i].cgroup_id = 0; qa.sub_sched_ctxs[i].weight = 100; cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids); @@ -2208,6 +2232,16 @@ void BPF_STRUCT_OPS(qmap_sub_ecaps_updated, s32 cid, u64 before, u64 after) execute_partition(); } +void BPF_STRUCT_OPS(qmap_sub_child_ecaps_updated, u64 cgroup_id, s32 cid, u64 before, + u64 after) +{ + __sync_fetch_and_add(&qa.nr_child_ecaps, 1); + + /* a child's last target must not outlive its PERF cap */ + if ((before & ~after) & SCX_CAP_PERF) + scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE); +} + SCX_OPS_CID_DEFINE(qmap_ops, .flags = SCX_OPS_ENQ_EXITING | SCX_OPS_TID_TO_TASK, .select_cid = (void *)qmap_select_cid, @@ -2232,6 +2266,7 @@ SCX_OPS_CID_DEFINE(qmap_ops, .sub_caps_updated = (void *)qmap_sub_caps_updated, .sub_ecaps_updated = (void *)qmap_sub_ecaps_updated, .sub_cid_sched_updated = (void *)qmap_sub_cid_sched_updated, + .sub_child_ecaps_updated = (void *)qmap_sub_child_ecaps_updated, .init_cids = (void *)qmap_init_cids, .init = (void *)qmap_init, .exit = (void *)qmap_exit, diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c index e83d5e2147dd..17cf584a2a3d 100644 --- a/tools/sched_ext/scx_qmap.c +++ b/tools/sched_ext/scx_qmap.c @@ -113,6 +113,7 @@ struct hier_prev { u64 nr_enq_blocked; u64 nr_inject_attempts; u64 nr_rescue_dsp; + u64 nr_child_ecaps; }; /* current wall-clock time as "HH:MM:SS" for the startup and interval headers */ @@ -208,18 +209,20 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg } format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges)); - printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu blocked=+%llu inj=+%llu rescue=+%llu\n", + printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu blocked=+%llu inj=+%llu rescue=+%llu child_ecaps=+%llu\n", (unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr, (unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap), (unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed), (unsigned long long)(qa->nr_enq_blocked - prev->nr_enq_blocked), (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts), - (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp)); + (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp), + (unsigned long long)(qa->nr_child_ecaps - prev->nr_child_ecaps)); prev->nr_reenq_cap = qa->nr_reenq_cap; prev->nr_reenq_immed = qa->nr_reenq_immed; prev->nr_enq_blocked = qa->nr_enq_blocked; prev->nr_inject_attempts = qa->nr_inject_attempts; prev->nr_rescue_dsp = qa->nr_rescue_dsp; + prev->nr_child_ecaps = qa->nr_child_ecaps; /* * alloc is the cid-time the partition handed each participant, and used diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h index 949459d06a18..9e6eefb39735 100644 --- a/tools/sched_ext/scx_qmap.h +++ b/tools/sched_ext/scx_qmap.h @@ -195,6 +195,7 @@ struct qmap_arena { u64 nr_enq_blocked; /* SCX_ENQ_BLOCKED dispatches */ u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */ u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */ + u64 nr_child_ecaps; /* ops.sub_child_ecaps_updated() deliveries */ u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */ }; From d1350c523f481769182604d758ecfcdf3597b402 Mon Sep 17 00:00:00 2001 From: Samuele Mariotti Date: Mon, 12 Oct 2026 13:59:28 +0200 Subject: [PATCH 53/69] selftests/sched_ext: Fix build error in dequeue_remote selftest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building the dequeue_remote selftest with newer compilers (e.g., gcc 16) triggers the following error: dequeue_remote.c:39:32: error: variable ‘sum’ set but not used The 'volatile' qualifier prevents the writes from being optimized away, but does not silence the warning, since 'sum' is indeed only written and never read. Consume 'sum' via an empty asm() with a register input constraint, as already done for the dequeue selftest in commit 3788e3251653 ("selftests/sched_ext: Fix build error in dequeue selftest"). This forces the compiler to keep the accumulated value (preserving the CPU stress loop) and avoids the build error. Signed-off-by: Samuele Mariotti Reviewed-by: Kuba Piecuch Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/dequeue_remote.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/testing/selftests/sched_ext/dequeue_remote.c b/tools/testing/selftests/sched_ext/dequeue_remote.c index f2c1901030de..af344581d5f1 100644 --- a/tools/testing/selftests/sched_ext/dequeue_remote.c +++ b/tools/testing/selftests/sched_ext/dequeue_remote.c @@ -43,6 +43,7 @@ static void worker_fn(int id) for (j = 0; j < 20000 + id * 1000; j++) sum += j; + asm volatile("" : : "r"(sum)); if (id & 1) usleep(100); else From 09d17543c3194ee68f2f2576a98d0db1f37de16d Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 14 Oct 2026 21:13:20 +0800 Subject: [PATCH 54/69] selftests/sched_ext: Fix reload_loop error-path use-after-free If the second pthread_create() fails, run() returns and the runner calls cleanup, which destroys the skeleton while the first reload thread is still running do_reload_loop() and dereferencing its maps - a use-after-free on the error path. Set force_exit and join the first thread before failing. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/reload_loop.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/sched_ext/reload_loop.c b/tools/testing/selftests/sched_ext/reload_loop.c index 49297b83d748..a00f3964dba0 100644 --- a/tools/testing/selftests/sched_ext/reload_loop.c +++ b/tools/testing/selftests/sched_ext/reload_loop.c @@ -53,7 +53,11 @@ static enum scx_test_status run(void *ctx) SCX_FAIL_IF(err, "Failed to create thread 0"); err = pthread_create(&threads[1], NULL, do_reload_loop, NULL); - SCX_FAIL_IF(err, "Failed to create thread 1"); + if (err) { + force_exit = true; + pthread_join(threads[0], NULL); + SCX_FAIL("Failed to create thread 1"); + } SCX_FAIL_IF(pthread_join(threads[0], &ret), "thread 0 failed"); SCX_FAIL_IF(pthread_join(threads[1], &ret), "thread 1 failed"); From 883c9e8104cfb522a227eeddd95f8077cb77d286 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 14 Oct 2026 21:13:21 +0800 Subject: [PATCH 55/69] sched_ext: Add SCX_ECODE_RSN_CGROUP_OFFLINE to user_exit_info.h The kernel-side scx_exit_code enum gained SCX_ECODE_RSN_CGROUP_OFFLINE in ebeca1f930ea ("sched_ext: Introduce cgroup sub-sched support"), which exits a scheduler when its cgroup goes offline, but the tools-side user_exit_info.h was not updated. Userspace cannot decode exit codes carrying this reason. Add the missing enum value. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- tools/sched_ext/include/scx/user_exit_info.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/sched_ext/include/scx/user_exit_info.h b/tools/sched_ext/include/scx/user_exit_info.h index 56a02b549aef..cf27ac2447ef 100644 --- a/tools/sched_ext/include/scx/user_exit_info.h +++ b/tools/sched_ext/include/scx/user_exit_info.h @@ -52,6 +52,7 @@ enum scx_exit_code { /* Reasons */ SCX_ECODE_RSN_HOTPLUG = 1LLU << 32, + SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32, /* Actions */ SCX_ECODE_ACT_RESTART = 1LLU << 48, From 0aeb67fee3cae416fa200d458a3010516ed426e9 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 14 Oct 2026 21:13:22 +0800 Subject: [PATCH 56/69] selftests/sched_ext: Fail loudly when EXIT_KIND is missing from BTF EXIT_KIND() resolved scx_exit_kind values through __COMPAT_ENUM_OR_ZERO(), which maps a missing enumerator to 0 (== SCX_EXIT_NONE). struct scx_exit_info carries the enum itself, so the BTF of any loaded scheduler always has the type, and every enumerator the tests resolve today exists on all sched_ext kernels: the fallback got the value right, and the single comparison that could pass vacuously was the kick.c error poll. Still, a silently-zeroed lookup is a trap. EXIT_KIND() is now an alias of SCX_KIND_VAL(), which fails the test loudly when the enumerator is missing. Convert the remaining `kind != EXIT_KIND(SCX_EXIT_NONE)` checks (allowed_cpus, cgroup_nr_cpus, cyclic_kick_wait and enq_blocked) to UEI_EXITED() so they keep their semantics without a BTF lookup, and hoist kick.c invalid_one()'s error_kind resolution above the gated-worker spawn so a failing lookup cannot bypass its cleanup path. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- .../selftests/sched_ext/allowed_cpus.c | 2 +- .../selftests/sched_ext/cgroup_nr_cpus.c | 2 +- .../selftests/sched_ext/cyclic_kick_wait.c | 2 +- .../testing/selftests/sched_ext/enq_blocked.c | 2 +- tools/testing/selftests/sched_ext/kick.c | 8 ++++--- tools/testing/selftests/sched_ext/nohz_tick.c | 6 ++--- tools/testing/selftests/sched_ext/scx_test.h | 22 ++++++++++++++----- 7 files changed, 28 insertions(+), 16 deletions(-) diff --git a/tools/testing/selftests/sched_ext/allowed_cpus.c b/tools/testing/selftests/sched_ext/allowed_cpus.c index 773699d120ea..76311728a6b1 100644 --- a/tools/testing/selftests/sched_ext/allowed_cpus.c +++ b/tools/testing/selftests/sched_ext/allowed_cpus.c @@ -149,7 +149,7 @@ static enum scx_test_status run(void *ctx) /* Just sleeping is fine, plenty of scheduling events happening. */ sleep(1); - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + if (UEI_EXITED(skel, uei)) { SCX_ERR("Scheduler exited unexpectedly"); goto out; } diff --git a/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c index 4973f7080353..1475a8b3f05f 100644 --- a/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c +++ b/tools/testing/selftests/sched_ext/cgroup_nr_cpus.c @@ -400,7 +400,7 @@ static enum scx_test_status run(void *arg) !wait_nr_cpus(&ctx, ctx.leaf, nr_child, "leaf after cpuset disable")) goto out; - if (ctx.skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + if (UEI_EXITED(ctx.skel, uei)) { SCX_ERR("Scheduler exited unexpectedly"); goto out; } diff --git a/tools/testing/selftests/sched_ext/cyclic_kick_wait.c b/tools/testing/selftests/sched_ext/cyclic_kick_wait.c index c2e5aa9de715..178d1d75f332 100644 --- a/tools/testing/selftests/sched_ext/cyclic_kick_wait.c +++ b/tools/testing/selftests/sched_ext/cyclic_kick_wait.c @@ -152,7 +152,7 @@ static enum scx_test_status run(void *ctx) sleep(5); - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + if (UEI_EXITED(skel, uei)) { SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", (unsigned long long)skel->data->uei.kind, (long long)skel->data->uei.exit_code); diff --git a/tools/testing/selftests/sched_ext/enq_blocked.c b/tools/testing/selftests/sched_ext/enq_blocked.c index 26204548bbf0..1d5e090b54bc 100644 --- a/tools/testing/selftests/sched_ext/enq_blocked.c +++ b/tools/testing/selftests/sched_ext/enq_blocked.c @@ -844,7 +844,7 @@ static enum scx_test_status run_one(bool enq_blocked, bool cross_cpu, } } - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + if (UEI_EXITED(skel, uei)) { SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", (unsigned long long)skel->data->uei.kind, (long long)skel->data->uei.exit_code); diff --git a/tools/testing/selftests/sched_ext/kick.c b/tools/testing/selftests/sched_ext/kick.c index 7f01602c75b2..40c91d54c1eb 100644 --- a/tools/testing/selftests/sched_ext/kick.c +++ b/tools/testing/selftests/sched_ext/kick.c @@ -97,7 +97,7 @@ static bool wait_for_state(struct kick *skel, u32 wanted) for (i = 0; i < WAIT_LOOPS; i++) { if (__atomic_load_n(&skel->bss->state, __ATOMIC_ACQUIRE) == wanted) return true; - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) + if (UEI_EXITED(skel, uei)) return false; usleep(1000); } @@ -111,7 +111,7 @@ static bool wait_for_counter(struct kick *skel, const u64 *counter, u64 wanted) for (i = 0; i < WAIT_LOOPS; i++) { if (__atomic_load_n(counter, __ATOMIC_ACQUIRE) >= wanted) return true; - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) + if (UEI_EXITED(skel, uei)) return false; usleep(1000); } @@ -421,7 +421,9 @@ static enum scx_test_status invalid_one(struct kick_ctx *ctx, u32 scenario) enum scx_test_status ret = SCX_TEST_FAIL; int cpu = ctx->target_cpu; int i; + u64 error_kind; + error_kind = EXIT_KIND(SCX_EXIT_ERROR); victim = scx_test_spawn_gated_worker(cpu, false); if (victim.pid < 0) return SCX_TEST_FAIL; @@ -440,7 +442,7 @@ static enum scx_test_status invalid_one(struct kick_ctx *ctx, u32 scenario) goto out; for (i = 0; i < WAIT_LOOPS; i++) { - if (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_ERROR)) { + if (UEI_EXITED(skel, uei) == error_kind) { ret = SCX_TEST_PASS; break; } diff --git a/tools/testing/selftests/sched_ext/nohz_tick.c b/tools/testing/selftests/sched_ext/nohz_tick.c index df40976b9166..e19a97034b7f 100644 --- a/tools/testing/selftests/sched_ext/nohz_tick.c +++ b/tools/testing/selftests/sched_ext/nohz_tick.c @@ -165,7 +165,7 @@ static bool wait_for_counter(struct nohz_tick *skel, const u64 *counter, u64 val for (elapsed = 0; elapsed < timeout_ms; elapsed++) { if (__atomic_load_n(counter, __ATOMIC_RELAXED) >= value) return true; - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) + if (UEI_EXITED(skel, uei)) return false; usleep(1000); } @@ -182,7 +182,7 @@ static bool wait_for_tick_stop(struct nohz_tick *skel, const u64 *counter, int t u64 curr; usleep(1000); - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) + if (UEI_EXITED(skel, uei)) return false; curr = __atomic_load_n(counter, __ATOMIC_RELAXED); if (curr == prev) { @@ -461,7 +461,7 @@ static enum scx_test_status run(void *ctx_ptr) scx_test_stop_gated_worker(&challenger); check_exit: - if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_NONE)) { + if (UEI_EXITED(skel, uei)) { SCX_ERR("Scheduler exited unexpectedly (kind=%llu code=%lld)", (unsigned long long)skel->data->uei.kind, (long long)skel->data->uei.exit_code); diff --git a/tools/testing/selftests/sched_ext/scx_test.h b/tools/testing/selftests/sched_ext/scx_test.h index 90b8d6915bb7..319e67f19b6d 100644 --- a/tools/testing/selftests/sched_ext/scx_test.h +++ b/tools/testing/selftests/sched_ext/scx_test.h @@ -18,8 +18,6 @@ enum scx_test_status { SCX_TEST_FAIL, }; -#define EXIT_KIND(__ent) __COMPAT_ENUM_OR_ZERO("scx_exit_kind", #__ent) - struct scx_test { /** * name - The name of the testcase. @@ -119,13 +117,25 @@ void scx_test_register(struct scx_test *test); (s64)__val; \ }) +/* + * SCX_KIND_VAL - Resolve an scx_exit_kind enumerator from kernel BTF. + * + * Unlike __COMPAT_ENUM_OR_ZERO(), a missing enumerator fails the test + * loudly instead of silently resolving to 0 (== SCX_EXIT_NONE), which + * would make the comparison vacuous. + */ #define SCX_KIND_VAL(__kind) ({ \ - u64 __val = 0; \ - bool __found = false; \ + u64 __val = 0; \ \ - __found = __COMPAT_read_enum("scx_exit_kind", #__kind, &__val); \ - SCX_ASSERT(__found); \ + if (!__COMPAT_read_enum("scx_exit_kind", #__kind, &__val)) \ + SCX_FAIL("scx_exit_kind.%s not found in kernel BTF", #__kind); \ __val; \ }) +/* + * EXIT_KIND - Alias of SCX_KIND_VAL() for resolving scx_exit_kind + * enumerator values, kept for readability at the EXIT_KIND() call sites. + */ +#define EXIT_KIND(__ent) SCX_KIND_VAL(__ent) + #endif // # __SCX_TEST_H__ From bb905626a48c0da2268030871b1cb1a1e7cc78fd Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 14 Oct 2026 21:13:23 +0800 Subject: [PATCH 57/69] selftests/sched_ext: Add SCX_ASSERT_ALIVE and adopt it A scheduler that dies mid-test freezes the state that the test is about to assert on, so the assertions can pass vacuously. Add SCX_ASSERT_ALIVE(), which fails the test with the recorded exit kind, exit code and message, and use it in maximal, create_dsq and init_enable_count. Existing open-coded checks keep their goto-based cleanup and are not converted. init_enable_count also clears the uei between its two attach cycles, since the clean unregister of the first cycle would otherwise trip the check of the second. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- .../selftests/sched_ext/create_dsq.bpf.c | 8 ++++++++ tools/testing/selftests/sched_ext/create_dsq.c | 2 ++ .../sched_ext/init_enable_count.bpf.c | 8 ++++++++ .../selftests/sched_ext/init_enable_count.c | 13 +++++++++++++ .../testing/selftests/sched_ext/maximal.bpf.c | 6 +++++- tools/testing/selftests/sched_ext/maximal.c | 2 ++ tools/testing/selftests/sched_ext/scx_test.h | 18 ++++++++++++++++++ 7 files changed, 56 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/sched_ext/create_dsq.bpf.c b/tools/testing/selftests/sched_ext/create_dsq.bpf.c index 680cc4b6d8c7..24dacd1aab0c 100644 --- a/tools/testing/selftests/sched_ext/create_dsq.bpf.c +++ b/tools/testing/selftests/sched_ext/create_dsq.bpf.c @@ -12,6 +12,8 @@ char _license[] SEC("license") = "GPL"; u32 nr_lifecycle_tests; +UEI_DEFINE(uei); + void BPF_STRUCT_OPS(create_dsq_exit_task, struct task_struct *p, struct scx_exit_task_args *args) { @@ -84,10 +86,16 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(create_dsq_init) return 0; } +void BPF_STRUCT_OPS(create_dsq_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + SEC(".struct_ops.link") struct sched_ext_ops create_dsq_ops = { .init_task = (void *) create_dsq_init_task, .exit_task = (void *) create_dsq_exit_task, .init = (void *) create_dsq_init, + .exit = (void *) create_dsq_exit, .name = "create_dsq", }; diff --git a/tools/testing/selftests/sched_ext/create_dsq.c b/tools/testing/selftests/sched_ext/create_dsq.c index 422e6532ee71..84143d5710f4 100644 --- a/tools/testing/selftests/sched_ext/create_dsq.c +++ b/tools/testing/selftests/sched_ext/create_dsq.c @@ -35,6 +35,8 @@ static enum scx_test_status run(void *ctx) return SCX_TEST_FAIL; } + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); SCX_EQ(skel->bss->nr_lifecycle_tests, 1024); diff --git a/tools/testing/selftests/sched_ext/init_enable_count.bpf.c b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c index 5eb9edb1837d..c0a5dd8fe183 100644 --- a/tools/testing/selftests/sched_ext/init_enable_count.bpf.c +++ b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c @@ -15,6 +15,8 @@ char _license[] SEC("license") = "GPL"; u64 init_task_cnt, exit_task_cnt, enable_cnt, disable_cnt; u64 init_fork_cnt, init_transition_cnt; +UEI_DEFINE(uei); + s32 BPF_STRUCT_OPS_SLEEPABLE(cnt_init_task, struct task_struct *p, struct scx_init_task_args *args) { @@ -43,11 +45,17 @@ void BPF_STRUCT_OPS(cnt_disable, struct task_struct *p) __sync_fetch_and_add(&disable_cnt, 1); } +void BPF_STRUCT_OPS(cnt_exit, struct scx_exit_info *ei) +{ + UEI_RECORD(uei, ei); +} + SEC(".struct_ops.link") struct sched_ext_ops init_enable_count_ops = { .init_task = (void *) cnt_init_task, .exit_task = (void *) cnt_exit_task, .enable = (void *) cnt_enable, .disable = (void *) cnt_disable, + .exit = (void *) cnt_exit, .name = "init_enable_count", }; diff --git a/tools/testing/selftests/sched_ext/init_enable_count.c b/tools/testing/selftests/sched_ext/init_enable_count.c index 44577e30e764..c4a57038a1d6 100644 --- a/tools/testing/selftests/sched_ext/init_enable_count.c +++ b/tools/testing/selftests/sched_ext/init_enable_count.c @@ -5,6 +5,7 @@ * Copyright (c) 2023 Tejun Heo */ #include +#include #include #include #include @@ -72,7 +73,17 @@ static enum scx_test_status run_test(bool global) close(pipe_fds[1]); signal(SIGCHLD, SIG_DFL); + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); + + /* + * The clean unregister above is recorded in uei and would + * otherwise trip the liveness check of the second cycle, which + * reuses this skeleton. + */ + memset(&skel->data->uei, 0, sizeof(skel->data->uei)); + SCX_GE(skel->bss->init_task_cnt, num_pre_forks); SCX_GE(skel->bss->exit_task_cnt, num_pre_forks); @@ -124,6 +135,8 @@ static enum scx_test_status run_test(bool global) status); } + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); SCX_GE(skel->bss->init_task_cnt, 2 * num_children); diff --git a/tools/testing/selftests/sched_ext/maximal.bpf.c b/tools/testing/selftests/sched_ext/maximal.bpf.c index 04a369078aac..9ea664e2ff70 100644 --- a/tools/testing/selftests/sched_ext/maximal.bpf.c +++ b/tools/testing/selftests/sched_ext/maximal.bpf.c @@ -14,6 +14,8 @@ char _license[] SEC("license") = "GPL"; #define DSQ_ID 0 +UEI_DEFINE(uei); + s32 BPF_STRUCT_OPS(maximal_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { @@ -132,7 +134,9 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(maximal_init) } void BPF_STRUCT_OPS(maximal_exit, struct scx_exit_info *info) -{} +{ + UEI_RECORD(uei, info); +} SEC(".struct_ops.link") struct sched_ext_ops maximal_ops = { diff --git a/tools/testing/selftests/sched_ext/maximal.c b/tools/testing/selftests/sched_ext/maximal.c index 1dc369224670..6c6b80e0446e 100644 --- a/tools/testing/selftests/sched_ext/maximal.c +++ b/tools/testing/selftests/sched_ext/maximal.c @@ -35,6 +35,8 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.maximal_ops); SCX_FAIL_IF(!link, "Failed to attach scheduler"); + SCX_ASSERT_ALIVE(skel, uei); + bpf_link__destroy(link); return SCX_TEST_PASS; diff --git a/tools/testing/selftests/sched_ext/scx_test.h b/tools/testing/selftests/sched_ext/scx_test.h index 319e67f19b6d..9e102d7f2893 100644 --- a/tools/testing/selftests/sched_ext/scx_test.h +++ b/tools/testing/selftests/sched_ext/scx_test.h @@ -108,6 +108,24 @@ void scx_test_register(struct scx_test *test); #define SCX_ASSERT(_x) SCX_FAIL_IF(!(_x), "Expected %s to be true (%lu)", \ #_x, (u64)(_x)) +/* + * SCX_ASSERT_ALIVE - Fail the test if the scheduler under test exited + * on its own between attach and this check, e.g. from a runtime error. + * A scheduler that dies mid-test freezes whatever state the test is + * about to assert on, which can make the assertions pass vacuously. + * uei.msg carries the actual error text, so print it alongside the + * recorded kind and exit code. + */ +#define SCX_ASSERT_ALIVE(__skel, __uei) \ + do { \ + if (UEI_EXITED(__skel, __uei)) \ + SCX_FAIL("Scheduler exited unexpectedly" \ + " (kind=%llu code=%lld msg='%s')", \ + (unsigned long long)(__skel)->data->__uei.kind, \ + (long long)(__skel)->data->__uei.exit_code, \ + (__skel)->data->__uei.msg); \ + } while (0) + #define SCX_ECODE_VAL(__ecode) ({ \ u64 __val = 0; \ bool __found = false; \ From cce3a39cebca074e0994e2fbce7a6062b80e42d8 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 14 Oct 2026 21:13:24 +0800 Subject: [PATCH 58/69] selftests/sched_ext: Bound exit waits exit, prog_run and hotplug wait for the scheduler under test to exit with no time bound; a scheduler that never exits hangs the runner instead of failing the test. Bound the waits to ~10s (iteration-based) with the new SCX_TIMED_WAIT_UEI() helper, which stretches on a loaded system instead of failing early. exit.c returns straight from its timeout path, which would leave the link and the skeleton alive and the unexited scheduler attached for the rest of the runner run - every later test would then fail to attach with EBUSY and bury the real failure. Destroy both before failing, like prog_run and hotplug already destroy the link. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- tools/testing/selftests/sched_ext/exit.c | 8 ++++++-- tools/testing/selftests/sched_ext/hotplug.c | 12 ++++++++---- tools/testing/selftests/sched_ext/prog_run.c | 7 +++++-- tools/testing/selftests/sched_ext/scx_test.h | 17 +++++++++++++++++ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c index 01b17092d5c8..fd6f1c9ab2db 100644 --- a/tools/testing/selftests/sched_ext/exit.c +++ b/tools/testing/selftests/sched_ext/exit.c @@ -43,8 +43,12 @@ static enum scx_test_status run(void *ctx) } /* Assumes uei.kind is written last */ - while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + bpf_link__destroy(link); + exit__destroy(skel); + return SCX_TEST_FAIL; + } SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF)); SCX_EQ(skel->data->uei.exit_code, tc); diff --git a/tools/testing/selftests/sched_ext/hotplug.c b/tools/testing/selftests/sched_ext/hotplug.c index 10b8d42bd89b..3c07686a93c8 100644 --- a/tools/testing/selftests/sched_ext/hotplug.c +++ b/tools/testing/selftests/sched_ext/hotplug.c @@ -85,8 +85,10 @@ static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined) if (toggle_online_status(onlining ? 1 : 0)) goto out_destroy_link; - while (!UEI_EXITED(skel, uei)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + goto out_destroy_link; + } SCX_EQ(skel->data->uei.kind, kind); SCX_EQ(UEI_REPORT(skel, uei), code); @@ -130,8 +132,10 @@ static enum scx_test_status test_hotplug_attach(void) goto out_destroy_link; SCX_ASSERT(link); - while (!UEI_EXITED(skel, uei)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + goto out_destroy_link; + } kind = SCX_KIND_VAL(SCX_EXIT_UNREG_KERN); code = SCX_ECODE_VAL(SCX_ECODE_ACT_RESTART) | diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c index 1129ec2aaddc..369c744ac06e 100644 --- a/tools/testing/selftests/sched_ext/prog_run.c +++ b/tools/testing/selftests/sched_ext/prog_run.c @@ -55,8 +55,11 @@ static enum scx_test_status run(void *ctx) } /* Assumes uei.kind is written last */ - while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) - sched_yield(); + if (!SCX_TIMED_WAIT_UEI(skel, uei, 10000)) { + SCX_ERR("Timed out waiting for scheduler to exit"); + status = SCX_TEST_FAIL; + goto out; + } if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) { SCX_ERR("Unexpected exit kind: %llu", diff --git a/tools/testing/selftests/sched_ext/scx_test.h b/tools/testing/selftests/sched_ext/scx_test.h index 9e102d7f2893..8490b01fff74 100644 --- a/tools/testing/selftests/sched_ext/scx_test.h +++ b/tools/testing/selftests/sched_ext/scx_test.h @@ -126,6 +126,23 @@ void scx_test_register(struct scx_test *test); (__skel)->data->__uei.msg); \ } while (0) +/* + * SCX_TIMED_WAIT_UEI - Wait for the scheduler under test to exit, i.e. + * its uei to be filled in, for up to @__ms iterations at ~1ms each. + * Returns true if the scheduler exited and false on timeout, so a + * scheduler that never exits fails the test instead of hanging it. + * The bound is iteration-based rather than wall-clock and stretches + * on a loaded system instead of failing early. + */ +#define SCX_TIMED_WAIT_UEI(__skel, __uei, __ms) \ + ({ \ + s64 __left = (__ms); \ + \ + while (__left-- > 0 && !UEI_EXITED(__skel, __uei)) \ + usleep(1000); \ + UEI_EXITED(__skel, __uei) != 0; \ + }) + #define SCX_ECODE_VAL(__ecode) ({ \ u64 __val = 0; \ bool __found = false; \ From 6052353d2f32fd876ac0846b089f292dd5793141 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Wed, 14 Oct 2026 21:13:25 +0800 Subject: [PATCH 59/69] selftests/sched_ext: Report reload_loop attach failures reload_loop skips bpf_map__attach_struct_ops() failures silently: measured on a loaded host, 1125-1233 of 2048 iterations failed, all with EBUSY (the single scheduler slot changing hands between the two threads) - the test passed while roughly half of its iterations never ran. Count the failures per thread, recording any errno other than EBUSY separately. Fail the test on an unexpected errno or when a thread failed every attach; a thread that completed at least one attach has exercised the reload path, so ordinary EBUSY no longer prints on every run. Name the iteration and thread counts in an enum. Signed-off-by: Tao Cui Signed-off-by: Tejun Heo --- .../testing/selftests/sched_ext/reload_loop.c | 43 ++++++++++++++++--- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/sched_ext/reload_loop.c b/tools/testing/selftests/sched_ext/reload_loop.c index a00f3964dba0..7d8880fa1a03 100644 --- a/tools/testing/selftests/sched_ext/reload_loop.c +++ b/tools/testing/selftests/sched_ext/reload_loop.c @@ -4,6 +4,7 @@ * Copyright (c) 2024 David Vernet */ #include +#include #include #include #include @@ -11,8 +12,14 @@ #include "maximal.bpf.skel.h" #include "scx_test.h" +enum { + RELOAD_ITERS = 1024, + RELOAD_THREADS = 2, +}; + static struct maximal *skel; -static pthread_t threads[2]; +static pthread_t threads[RELOAD_THREADS]; +static int unexpected_errno; bool force_exit = false; @@ -31,22 +38,33 @@ static enum scx_test_status setup(void **ctx) static void *do_reload_loop(void *arg) { - u32 i; + u32 i, failures = 0; - for (i = 0; i < 1024 && !force_exit; i++) { + for (i = 0; i < RELOAD_ITERS && !force_exit; i++) { struct bpf_link *link; + errno = 0; link = bpf_map__attach_struct_ops(skel->maps.maximal_ops); - if (link) - bpf_link__destroy(link); + if (!link) { + int e = errno; + + if (e != EBUSY) + __atomic_store_n(&unexpected_errno, e, + __ATOMIC_RELAXED); + failures++; + continue; + } + + bpf_link__destroy(link); } - return NULL; + return (void *)(unsigned long)failures; } static enum scx_test_status run(void *ctx) { - int err; + unsigned long failures[RELOAD_THREADS] = {}; + int err, i; void *ret; err = pthread_create(&threads[0], NULL, do_reload_loop, NULL); @@ -60,7 +78,18 @@ static enum scx_test_status run(void *ctx) } SCX_FAIL_IF(pthread_join(threads[0], &ret), "thread 0 failed"); + failures[0] = (unsigned long)ret; + SCX_FAIL_IF(pthread_join(threads[1], &ret), "thread 1 failed"); + failures[1] = (unsigned long)ret; + + SCX_FAIL_IF(unexpected_errno, "unexpected attach errno %d", + unexpected_errno); + + for (i = 0; i < RELOAD_THREADS; i++) + SCX_FAIL_IF(failures[i] == RELOAD_ITERS, + "thread %d: every attach failed (%lu EBUSY)", + i, failures[i]); return SCX_TEST_PASS; } From 67d0d8d018f35beaa58a5e62a1d9c83414e570cd Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sat, 10 Oct 2026 11:06:21 +0200 Subject: [PATCH 60/69] sched_ext: Keep proxy donors with slice left on the local DSQ Commit ee172227d0dc ("sched_ext: Delegate proxy donor admission to BPF schedulers") makes put_prev_task_scx() pass a retained proxy donor to ops.enqueue() with SCX_ENQ_BLOCKED. Some of these puts are only proxy bookkeeping; proxy_resched_idle() drops the rq's donor reference before switching to idle when: - find_proxy_task() cannot run the mutex owner yet and retries through idle (e.g., the owner is on a remote CPU), - proxy_migrate_task() detaches the donor before migration, - proxy_deactivate() blocks a task in the chain whose owner cannot run. In the first case, BPF has already selected a donor with slice left. Returning it to ops.enqueue() forces BPF to dispatch it again before proxy resolution can continue. In the other two cases, the donor is typically deactivated right after ops.enqueue(), undoing any placement BPF makes. As also discussed at the sched_ext microconference at Linux Plumbers 2026, these ops.enqueue/dequeue() invocations do not represent any meaningful scheduling events, so we should avoid triggering them. A more reasonable semantic is to keep a donor with slice left at the head of the local DSQ, so the next pick can resolve its owner, or deactivation can remove it without an unnecessary BPF handoff. An IMMED donor can stay there only for a bookkeeping put during proxy resolution. Mark a blocked pick as awaiting resolution with SCX_RQ_PROXY_PENDING to distinguish that put from a real preemption and re-insert the donor with SCX_ENQ_IMMED, so that it only needs SCX_CAP_ENQ_IMMED rather than SCX_CAP_ENQ. A preempted IMMED donor still returns to BPF. If a higher priority class takes the CPU once proxy resolution completes, schedule a local reenqueue, so that a parked IMMED donor returns to BPF instead of lingering on the local DSQ. Update the SCX_ENQ_IMMED and SCX_ENQ_BLOCKED documentation accordingly. Moreover, a donor that has run out of slice now follows the regular put path, so it can receive SCX_ENQ_LAST when the CPU is about to go idle and the scheduler needs to arrange a follow-up scheduling event. This does not apply to bookkeeping puts, where the switch to idle is only temporary. The blocked-donor specific sanity checks go away together with the special case. When a retained donor is removed ahead of a scheduler ownership or policy change, mark the proxy reset with SCX_RQ_PROXY_BLOCKING and skip reenqueueing the active donor: sched_proxy_block_task() will dequeue it next. Report it as not runnable to ops.stopping(), as a regular dequeue of the current donor does. This avoids a transient ops.enqueue/dequeue() pair and a false ENQ_LAST warning. With this, the sched_ext core absorbs the transient proxy-bookkeeping puts and the BPF scheduler only sees a blocked donor when there is an actual placement decision to make. Fixes: ee172227d0dc ("sched_ext: Delegate proxy donor admission to BPF schedulers") Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 84 +++++++++++++++++++++++++++---------- kernel/sched/ext/internal.h | 10 ++++- kernel/sched/sched.h | 2 + 3 files changed, 72 insertions(+), 24 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index a2e0382ce3dd..dc5d91a051ae 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -24,6 +24,17 @@ DEFINE_RAW_SPINLOCK(scx_sched_lock); +/* + * Block a retained proxy donor, telling put_prev_task_scx() that the donor is + * about to be dequeued and must not be reenqueued. + */ +static void scx_block_proxy_donor(struct rq *rq, struct task_struct *p) +{ + rq->scx.flags |= SCX_RQ_PROXY_BLOCKING; + sched_proxy_block_task(rq, p); + rq->scx.flags &= ~SCX_RQ_PROXY_BLOCKING; +} + bool __scx_allow_proxy_exec(const struct task_struct *p) { struct scx_sched *sch; @@ -47,7 +58,7 @@ void scx_prepare_task_sched_change(struct task_struct *p) lockdep_assert_rq_held(task_rq(p)); update_rq_clock(task_rq(p)); - sched_proxy_block_task(task_rq(p), p); + scx_block_proxy_donor(task_rq(p), p); } /* @@ -1189,6 +1200,14 @@ void scx_proxy_reenqueue_retry(struct rq *rq, struct task_struct *next) scx_proxy_update_tick(rq, next); #endif + /* + * A higher priority class may wake up while @rq is transiently idle + * for proxy resolution. wakeup_preempt_scx() isn't invoked in that + * case, so reenqueue IMMED donors parked on the local DSQ here. + */ + if (rq->scx.nr_immed && sched_class_above(rq->next_class, &ext_sched_class)) + scx_schedule_reenq_local(rq, 0); + if (rq->scx.flags & SCX_RQ_PROXY_RETRY) { rq->scx.flags &= ~SCX_RQ_PROXY_RETRY; schedule_deferred_locked(rq); @@ -3362,6 +3381,16 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t bool first = type == SNT_PICK; bool can_stop_tick; + /* + * A blocked pick is provisional until proxy resolution completes, see + * put_prev_task_scx(). A retry can repick the same donor, so + * SNT_REPICK must set it again. + */ + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + if (sched_proxy_exec() && p->is_blocked && + (type == SNT_PICK || type == SNT_REPICK)) + rq->scx.flags |= SCX_RQ_PROXY_PENDING; + if (type == SNT_REPICK) return; @@ -3441,6 +3470,7 @@ void scx_proxy_donor_start(struct rq *rq) struct task_struct *donor = rq->donor; lockdep_assert_rq_held(rq); + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; if (donor->sched_class == &ext_sched_class && (donor->scx.flags & SCX_TASK_QUEUED)) scx_start_task_running(rq, donor); @@ -3501,8 +3531,13 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, struct task_struct *next) { struct scx_sched *sch = scx_task_sched(p); + bool proxy_put = p->is_blocked && next == rq->idle && + (rq->scx.flags & SCX_RQ_PROXY_PENDING); + bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING; bool rescue_keep = false; + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -3526,31 +3561,24 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, if (next != p && (p->scx.flags & SCX_TASK_QUEUED) && (p->scx.flags & SCX_TASK_RUN_TRACKED)) { if (SCX_HAS_OP(sch, stopping)) - SCX_CALL_OP_TASK(sch, stopping, rq, p, true); + SCX_CALL_OP_TASK(sch, stopping, rq, p, !proxy_block); p->scx.flags &= ~SCX_TASK_RUN_TRACKED; } if (p->scx.flags & SCX_TASK_QUEUED) { + if (proxy_block) + goto switch_class; + set_task_runnable(rq, p); - /* Delegate retained donor admission to its owning BPF scheduler. */ - if (p->is_blocked) { - /* - * If the donor is the same and only the mutex owner - * changes, avoid triggering another ops.enqueue(): the - * BPF scheduler has already admitted the donor, so it - * can continue running. - */ - if (next == p) - goto switch_class; - - if (WARN_ON_ONCE(!sch)) - goto switch_class; - WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_BLOCKED)); - scx_do_enqueue_task(rq, p, 0, -1); + /* + * If the donor is the same and only the mutex owner changes, + * avoid triggering another ops.enqueue(): the BPF scheduler has + * already admitted the donor, so it can continue running. + */ + if (p->is_blocked && next == p) goto switch_class; - } /* * If @p has slice left and is being put, @p is getting @@ -3559,12 +3587,16 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * DSQ unless it was an IMMED task. IMMED tasks should not * linger on a busy CPU, reenqueue them to the BPF scheduler. * + * The exception is an IMMED donor put by proxy_resched_idle(): + * the CPU isn't busy, proxy resolution is only retrying through + * idle, so keep the donor where it can be picked again. + * * An open rescue must keep @p on the local DSQ even if the * scheduler zeroed the slice in ops.stopping() above. */ if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && !scx_bypassing(sch, cpu_of(rq))) { - if (p->scx.flags & SCX_TASK_IMMED) { + if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); } else { @@ -3581,6 +3613,9 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, enq_flags |= SCX_ENQ_HEAD; } else { enq_flags |= SCX_ENQ_HEAD; + /* only require SCX_CAP_ENQ_IMMED, see scx_caps_for_enq() */ + if (proxy_put && (p->scx.flags & SCX_TASK_IMMED)) + enq_flags |= SCX_ENQ_IMMED; } scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, @@ -3593,13 +3628,15 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * If @p is runnable but we're about to enter a lower * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu. - * This doesn't apply if the baseline access on the CPU is lost. + * This doesn't apply if the baseline access on the CPU is lost or + * proxy resolution temporarily switches to idle. * * Under core scheduling, a pick dispatches only when nothing is * locally runnable and can legitimately go idle with @p still * runnable (see do_pick_task_scx()). */ - if (next && sched_class_above(&ext_sched_class, next->sched_class) && + if (!proxy_put && next && + sched_class_above(&ext_sched_class, next->sched_class) && scx_task_can_stay_on_cpu(rq, p)) { WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); @@ -3781,6 +3818,9 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) enum scx_dsp_verdict verdict; struct task_struct *p; + /* A retry can abandon a provisional blocked-donor pick. */ + rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; + /* see kick_sync_wait_bal_cb() */ smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1); @@ -4801,7 +4841,7 @@ void scx_prepare_setscheduler(struct task_struct *p, int policy) lockdep_assert_rq_held(task_rq(p)); if (scx_enabled() && p->policy != policy && policy == SCHED_EXT) - sched_proxy_block_task(task_rq(p), p); + scx_block_proxy_donor(task_rq(p), p); } static void process_ddsp_deferred_locals(struct rq *rq) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index e1afcd4e043f..f6a2ee3ca172 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1842,8 +1842,8 @@ enum scx_enq_flags { /* * Only allowed on local DSQs. Guarantees that the task either gets * on the CPU immediately and stays on it, or gets reenqueued back - * to the BPF scheduler. It will never linger on a local DSQ or be - * silently put back after preemption. + * to the BPF scheduler. A blocked proxy donor may be put back on the + * local DSQ temporarily while proxy execution resolves its mutex owner. * * The protection persists until the next fresh enqueue - it * survives SAVE/RESTORE cycles, slice extensions and preemption. @@ -1890,6 +1890,12 @@ enum scx_enq_flags { /* * The task is blocked on a mutex and is being kept runnable as a proxy * donor. Only passed to ops.enqueue() when %SCX_OPS_ENQ_BLOCKED is set. + * + * Blocking on the mutex does not enqueue the task by itself. A donor put + * with slice left stays at the head of the local DSQ, except when an + * IMMED donor is preempted or cannot stay local. It is passed to + * ops.enqueue() when its slice runs out, the IMMED placement cannot be + * kept, or proxy execution moves it to the owner's CPU. */ SCX_ENQ_BLOCKED = 1LLU << 42, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 7a5739acba5f..37d4808d7f19 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -793,6 +793,8 @@ enum scx_rq_flags { SCX_RQ_ROOT_IDLE_RENOTIFY = 1 << 8, /* the root is owed update_idle() */ SCX_RQ_PROXY_RETRY = 1 << 9, /* proxy-rejected tasks need retry */ SCX_RQ_PROXY_TICK = 1 << 10, /* proxy execution requires the tick */ + SCX_RQ_PROXY_PENDING = 1 << 11, /* blocked donor awaits proxy resolution */ + SCX_RQ_PROXY_BLOCKING = 1 << 12, /* donor is being removed for sched change */ SCX_RQ_IN_WAKEUP = 1 << 16, SCX_RQ_IN_DISPATCH = 1 << 17, From 43e1ef14a47257ebf7e3bb3ab9a6df9c6f817c5f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 15 Oct 2026 07:18:05 -1000 Subject: [PATCH 61/69] sched_ext: Fix use-after-free of a local reenq node left on an inactive cpu A local DSQ reenq is queued on the rq through a per-sched node and run from deferred work. When the reenq is requested from dispatch, the work is a balance callback queued once dispatch is done. A cpu going down installs balance push early in its teardown, and queue_balance_callback() refuses new callbacks from then on. ops.dispatch() keeps running on that cpu until the rq goes offline after an RCU grace period, so a reenq it requests in that window may never run. The IMMED handling in dispatch_one() requests one the same way. A reenq requested from elsewhere for a cpu that is already offline goes through irq_work_queue_on(), which warns, and its work doesn't run until the cpu comes back. Either way the node stays on the rq's list. When the scheduler is freed, scx_sched_free_rcu_work() warns and frees the pcpu with the node still linked, and the list is walked again once the cpu comes back. An inactive cpu's local DSQ holds only tasks that are bound to it or are being moved off, so the reenq is pointless. Ignore reenqs of an inactive cpu's local DSQ. The test is lockless. The hotplug thread of a cpu going down runs on that cpu, and it marks the cpu inactive before it installs balance push. A dispatch on the cpu runs to completion before that thread can run there, so a dispatch that saw the cpu active has queued its callback by then. Requests from outside dispatch run from the wakeup path or through irq_work, which balance push doesn't affect. The grace period after the cpu goes inactive waits for any requester that saw it active, so that irq_work is queued while the cpu is online. Fixes: 238eba8c210d ("sched_ext: Use schedule_deferred_locked() in schedule_dsq_reenq()") Fixes: 76edc2761ab8 ("sched_ext: Use irq_work_queue_on() in schedule_deferred()") Cc: stable@vger.kernel.org # v7.1+ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi Tested-by: Andrea Righi --- kernel/sched/ext/ext.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 91120fc17d30..2c213153bdab 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1102,6 +1102,18 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, if (dsq->id == SCX_DSQ_LOCAL) { rq = container_of(dsq, struct rq, scx.local_dsq); + /* + * An inactive cpu's local DSQ holds only tasks that are bound + * to it or are being moved off, so a reenq of it is pointless. + * Once balance push is installed, queue_balance_callback() also + * drops the callback that would run a reenq requested from + * dispatch, and irq_work_queue_on() can't reach an offline cpu. + * The reenq's node would then stay on this rq's list after + * @sch's pcpu is freed. + */ + if (unlikely(!cpu_active(cpu_of(rq)))) + return; + /* * A sub-sched lacking baseline access on the target cid has no * business triggering IPIs. The lockless test is fine: slipping @@ -10020,7 +10032,8 @@ __bpf_kfunc struct task_struct *scx_bpf_dsq_peek(u64 dsq_id, * - Local DSQs (%SCX_DSQ_LOCAL or %SCX_DSQ_LOCAL_ON | $cpu) * - User DSQs * - * Re-enqueues are performed asynchronously. Can be called from anywhere. + * Re-enqueues are performed asynchronously. Can be called from anywhere. A + * re-enqueue of an inactive cpu's local DSQ is ignored. * * %SCX_DSQ_LOCAL resolves to the local DSQ of the rq the current scheduler * operation is locked to - e.g. the rq being dispatched for in ops.dispatch() - From 30fb2b1e4d41a641a059f295a717ffa0e24307e7 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 16 Oct 2026 19:37:16 -1000 Subject: [PATCH 62/69] sched_ext: Fix the running task keeping its protection into bypass Protection ends when the scheduler enters bypass, so that bypass scheduling can cut every task's slice. scx_bypass() clears it for the queued tasks by cycling them, and strips the running task's in the same walk over rq->scx.runnable_list. The running task is never on that list. set_next_task_scx() takes it off and put_prev_task_scx() puts it back, so the strip never runs. A protected running task then keeps its slice into bypass, where the tick's attempts to cut it are refused. Strip the running task's protection outside the walk. Fixes: 9cfc6ab34a31 ("sched_ext: Add SCX_TASK_PROTECTED") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2c213153bdab..d6689bf8e769 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6318,6 +6318,16 @@ void scx_bypass(struct scx_sched *sch, bool bypass) continue; } + /* + * Bypass trumps protection. Cycling the queued tasks below + * clears theirs. The current task isn't on the runnable list, + * so strip its protection here. + */ + p = rq->curr; + if (bypass && p->sched_class == &ext_sched_class && + scx_is_descendant(scx_task_sched(p), sch)) + scx_task_slice_ended(rq, p); + /* * The use of list_for_each_entry_safe_reverse() is required * because each task is going to be removed from and added back @@ -6330,13 +6340,6 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (!scx_is_descendant(scx_task_sched(p), sch)) continue; - /* - * Bypass trumps protection. Cycling clears for queued - * tasks but current task needs explicit stripping. - */ - if (bypass && task_current(rq, p)) - scx_task_slice_ended(rq, p); - /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { /* nothing */ ; From 5809f749fbdbaf9c6b34ecbe6d2b413e56eec6e3 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Fri, 16 Oct 2026 19:37:17 -1000 Subject: [PATCH 63/69] sched_ext: Fix a bypassed task stalling on a cpu its scheduler has no caps on A bypassing sub-scheduler's tasks are run by its nearest ancestor that isn't bypassing. The ancestor puts them on the local DSQ under its own caps. Whether such a task may stay on the cpu and whether a cap revoke reenqueues it are still checked against the caps of the task's own scheduler. The bypass load balancer moves tasks between cpus without looking at caps, so a bypassed task can run on a cpu where its own scheduler holds none. Once its slice runs out, it can't stay. It goes to the ancestor's bypass DSQ without a kick, and the cpu can go idle with it queued until something else reschedules the cpu. Make both checks against the caps of the scheduler running the task. When bypass lifts, the running task keeps its slice without a cap check, so evict it then if it can't stay on the cpu. Fixes: 75a8c8202c91 ("sched_ext: Add reject DSQ for cap-rejected dispatches") Fixes: 46a85ae6fe5b ("sched_ext: Tie cpu occupancy to SCX_CAP_BASE through the task slice") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 21 ++++++++++++--------- kernel/sched/ext/internal.h | 17 +++++++++++++++++ kernel/sched/ext/sub.c | 7 +++++-- kernel/sched/ext/sub.h | 6 +++++- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index d6689bf8e769..6b472506944e 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -322,7 +322,6 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) { -#ifdef CONFIG_EXT_SUB_SCHED /* * If @sch is a sub-sched which is bypassing, its tasks should go into * the bypass DSQs of the nearest ancestor which is not bypassing. The @@ -334,11 +333,7 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c * are re-enqueued after scx_bypassing() is turned on, guaranteeing that * all tasks are transferred to the right DSQs. */ - while (scx_parent(sch) && scx_bypassing(sch, cpu)) - sch = scx_parent(sch); -#endif /* CONFIG_EXT_SUB_SCHED */ - - return scx_bypass_dsq(sch, cpu); + return scx_bypass_dsq(scx_exec_sched(sch, cpu), cpu); } /** @@ -6322,11 +6317,19 @@ void scx_bypass(struct scx_sched *sch, bool bypass) * Bypass trumps protection. Cycling the queued tasks below * clears theirs. The current task isn't on the runnable list, * so strip its protection here. + * + * When bypass lifts, the current task keeps running on its + * slice without a cap check. Evict it if it can't stay on the + * cpu. */ p = rq->curr; - if (bypass && p->sched_class == &ext_sched_class && - scx_is_descendant(scx_task_sched(p), sch)) - scx_task_slice_ended(rq, p); + if (p->sched_class == &ext_sched_class && + scx_is_descendant(scx_task_sched(p), sch)) { + if (bypass) + scx_task_slice_ended(rq, p); + else if (!scx_task_can_stay_on_cpu(rq, p)) + scx_set_task_slice(p, 0); + } /* * The use of list_for_each_entry_safe_reverse() is required diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 352547aedbb7..03f03204d158 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2539,4 +2539,21 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; #endif /* CONFIG_EXT_SUB_SCHED */ +/** + * scx_exec_sched - Find the sched running @sch's tasks on @cpu + * @sch: sched owning the tasks + * @cpu: cpu of interest + * + * A bypassing sched's tasks are run by its nearest ancestor that isn't + * bypassing. Return @sch if it isn't bypassing on @cpu, otherwise that + * ancestor, or the root if every ancestor is bypassing. Hold @cpu's rq lock for + * a stable answer. + */ +static inline struct scx_sched *scx_exec_sched(struct scx_sched *sch, s32 cpu) +{ + while (scx_parent(sch) && scx_bypassing(sch, cpu)) + sch = scx_parent(sch); + return sch; +} + #endif /* _KERNEL_SCHED_EXT_INTERNAL_H */ diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index fee75fcdb59d..90430493bb72 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -755,18 +755,21 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r /* @p lost the caps needed to stay on @rq's local DSQ? Record reason if so. */ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { + s32 cpu = cpu_of(rq); u64 missing; /* migration-disabled tasks and the rescuee are admitted capless */ if (is_migration_disabled(p) || p == scx_rescuee(rq)) return false; - missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p)); + /* check caps against the sched running a bypassed task */ + missing = scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, + scx_caps_for_task(p)); if (likely(!missing)) return false; p->scx.reenq_reason_caps = missing; - p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu_of(rq)); + p->scx.reenq_reason_cid = __scx_cpu_to_cid(cpu); return true; } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index f7bcdfda8dd8..0ac7b8e59205 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -199,6 +199,8 @@ static inline u64 scx_caps_implied(u64 cap) /* may @p keep running on @rq's cpu? requires baseline cpu access */ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { + s32 cpu = cpu_of(rq); + if (!scx_has_subs()) return true; @@ -206,7 +208,9 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p if (unlikely(is_migration_disabled(p))) return true; - return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE)); + /* check caps against the sched running a bypassed task */ + return likely(!scx_missing_caps(scx_exec_sched(scx_task_sched(p), cpu), cpu, + SCX_CAP_BASE)); } /* the task admitted for rescue on @rq, NULL if none */ From ff0545e0c263727b200f60af47d0364e4d041bde Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sat, 17 Oct 2026 17:22:26 +0200 Subject: [PATCH 64/69] sched_ext: Don't leave balance callbacks queued across a pick retry The dispatch path can request deferred work to be run from a balance callback and pick_task_scx() queues rq->scx.deferred_bal_cb right after dispatching. However, a balance callback must be consumed before the rq lock is released and pick_task_scx() is not necessarily the end of the pick: - it returns RETRY_TASK when a higher priority class task has been enqueued while the dispatch had the rq lock released; - the dl_server pick can return NULL, in which case the lower classes are picked next. In both cases the callback stays queued while the core keeps picking, and the rq lock can be released again, either by prev_balance() or by another dispatch. For example, with an RT task as prev: __schedule() __pick_next_task() pick_task_scx() dispatch_pick() maybe_queue_balance_callback() <- deferred_bal_cb queued return RETRY_TASK prev_balance() balance_rt() pull_rt_task() double_lock_balance() <- rq lock released Any CPU acquiring the rq lock in that window finds a pending balance callback and triggers the following warning in rq_pin_lock(): WARNING: kernel/sched/sched.h:1919 at rq_lock+0x7f/0xe0, CPU#10: stress-ng-sleep/533 CPU: 10 UID: 0 PID: 533 Comm: stress-ng-sleep Not tainted 7.2.0-rc6-virtme #61 PREEMPT(full) Sched_ext: qmap (enabled+all) Call Trace: try_to_wake_up+0x2c1/0x5c0 hrtimer_wakeup+0x22/0x30 __hrtimer_run_queues+0x1d5/0x350 hrtimer_interrupt+0x106/0x1e0 __sysvec_apic_timer_interrupt+0x6f/0x210 sysvec_apic_timer_interrupt+0x71/0x90 This can be reproduced running scx_qmap -I together with a SCHED_FIFO workload that frequently sleeps, e.g.: $ vng -v --cpus 16 --memory 4G -- \ 'tools/sched_ext/build/bin/scx_qmap -I >/dev/null & \ stress-ng --sleep 4 --sched fifo --sched-prio 1 -t 30 -q' Fix this by queueing the balance callbacks only when the pick is final. Otherwise, defer the pending work to the irq_work path, which doesn't depend on the current rq lock section, and leave the kick sync wait pending for the next pick. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 73 ++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6b472506944e..2cd24a50f06c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3330,7 +3330,8 @@ static struct task_struct *first_local_task(struct rq *rq) } /* - * Run dispatch and queue the follow-up work for a pick. + * Run dispatch for a pick. The follow-up work is queued by do_pick_task_scx() + * once the pick is final. */ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, struct task_struct *prev) @@ -3340,18 +3341,6 @@ static enum scx_dsp_verdict dispatch_pick(struct rq *rq, struct rq_flags *rf, rq_unpin_lock(rq, rf); verdict = dispatch_one(rq, prev); rq_repin_lock(rq, rf); - maybe_queue_balance_callback(rq); - - /* - * Defer to a balance callback which can drop rq lock and enable IRQs. - * Waiting directly in the pick path would deadlock against CPUs sending - * us IPIs (e.g. TLB flushes) while we wait for them. - */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } return verdict; } @@ -3378,16 +3367,8 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r verdict = dispatch_one(rq, prev); - if (cpu_of(rq) == smp_processor_id()) { - maybe_queue_balance_callback(rq); - - /* see dispatch_pick() */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } - } else if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { + if (cpu_of(rq) != smp_processor_id() && + unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { /* * Balance callbacks must run in the context that queued them, * so they can't be queued on another CPU's rq. Run the deferred @@ -3430,8 +3411,10 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) else verdict = dispatch_pick(rq, rf, prev); - if (verdict == SCX_DSP_RETRY) - return RETRY_TASK; + if (verdict == SCX_DSP_RETRY) { + p = RETRY_TASK; + goto out; + } /* * If any higher-priority sched class enqueued a runnable task on this @@ -3441,8 +3424,10 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) * If @force_scx is true, always try to pick a SCHED_EXT task, * regardless of any higher-priority sched classes activity. */ - if (!force_scx && rq_modified_above(rq, &ext_sched_class)) - return RETRY_TASK; + if (!force_scx && rq_modified_above(rq, &ext_sched_class)) { + p = RETRY_TASK; + goto out; + } /* * If we're keeping @prev, replenish slice if necessary and keep running @@ -3458,7 +3443,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) } else { p = first_local_task(rq); if (!p) - return NULL; + goto out; if (unlikely(!p->scx.slice) && scx_task_can_stay_on_cpu(rq, p)) { struct scx_sched *sch = scx_task_sched(p); @@ -3473,6 +3458,38 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) } } +out: + if (cpu_of(rq) != smp_processor_id()) + return p; + + /* + * Balance callbacks must be consumed before the rq lock is released, so + * queue them only when the pick is final. On RETRY_TASK, or after a + * NULL pick from the dl_server, the core continues picking and can + * release the rq lock (e.g., prev_balance() or another dispatch). In + * that case, run the deferred work from irq_work, which doesn't depend + * on the current rq lock section, and leave the kick sync wait pending + * for the next pick. + */ + if (p != RETRY_TASK && (p || !force_scx)) { + maybe_queue_balance_callback(rq); + + /* + * Defer to a balance callback which can drop rq lock and enable + * IRQs. Waiting directly in the pick path would deadlock against + * CPUs sending us IPIs (e.g., TLB flushes) while we wait for + * them. + */ + if (unlikely(rq->scx.kick_sync_pending)) { + rq->scx.kick_sync_pending = false; + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, + kick_sync_wait_bal_cb); + } + } else if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) { + rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; + schedule_deferred(rq); + } + return p; } From 680edc7f23ad71e0ecce8a2364af3ae7f30dd11a Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sat, 17 Oct 2026 22:05:14 +0200 Subject: [PATCH 65/69] sched_ext: Don't queue balance callbacks from a core-sched pick Commit ff0545e0c263 ("sched_ext: Don't leave balance callbacks queued across a pick retry") queues the deferred work as a balance callback only when do_pick_task_scx() returns a final pick. However, under core scheduling, a pick that returns a task isn't final either: pick_next_task() goes on to pick the SMT siblings, and their dispatch can release the shared rq lock with the callback still queued, triggering the rq_pin_lock() warning. Moreover, dispatch_core_pick() returns early when another dispatch is in flight on the rq, and do_pick_task_scx() then consumes %SCX_RQ_BAL_CB_PENDING set by that dispatch while it's still running. This can be reproduced running scx_qmap -I on SMT siblings with coresched-tagged workloads that frequently sleep, mixed with untagged CPU-bound tasks. Fix this by always deferring the pending work to the irq_work path from dispatch_core_pick() under core scheduling, for both the local and the sibling picks, and by leaving %SCX_RQ_BAL_CB_PENDING alone in do_pick_task_scx(), so that it's handled by the dispatch that set it. Sample the core scheduling state once at the start of the pick, since core scheduling can get enabled while a non-core dispatch has the rq lock released, and the follow-up work must match the dispatch path that ran. Fixes: 4c95380701f5 ("sched/ext: Fold balance_scx() into pick_task_scx()") Cc: stable@vger.kernel.org # v6.19+ Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 2cd24a50f06c..e59e937534e9 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3367,15 +3367,20 @@ static enum scx_dsp_verdict dispatch_core_pick(struct rq *rq, struct rq_flags *r verdict = dispatch_one(rq, prev); - if (cpu_of(rq) != smp_processor_id() && - unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { - /* - * Balance callbacks must run in the context that queued them, - * so they can't be queued on another CPU's rq. Run the deferred - * work directly instead. - */ + /* + * A pick that returns a task isn't final under core scheduling: the + * selection goes on to pick the siblings, and their dispatch can + * release the rq lock, which can't happen with balance callbacks + * queued. Don't queue the deferred work as a balance callback, for this + * CPU or a sibling, and run it from irq_work instead. + * + * do_pick_task_scx() leaves %SCX_RQ_BAL_CB_PENDING alone under core + * scheduling, so that the early return above leaves it to the dispatch + * in flight, which handles it here. + */ + if (unlikely(rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; - run_deferred(rq); + schedule_deferred(rq); } rq_repin_lock(rq, rf); @@ -3398,6 +3403,7 @@ static struct task_struct * do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) { struct task_struct *prev = rq->curr; + bool core_sched = sched_core_enabled(rq); enum scx_dsp_verdict verdict; struct task_struct *p; @@ -3406,7 +3412,12 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) rq_modified_begin(rq, &ext_sched_class); - if (sched_core_enabled(rq)) + /* + * Core scheduling can get enabled while dispatch_pick() has the rq lock + * released. Stick to @core_sched, so that the follow-up work below + * matches the dispatch that ran. + */ + if (core_sched) verdict = dispatch_core_pick(rq, rf, prev); else verdict = dispatch_pick(rq, rf, prev); @@ -3472,7 +3483,9 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) * for the next pick. */ if (p != RETRY_TASK && (p || !force_scx)) { - maybe_queue_balance_callback(rq); + /* under core scheduling, see dispatch_core_pick() */ + if (!core_sched) + maybe_queue_balance_callback(rq); /* * Defer to a balance callback which can drop rq lock and enable @@ -3485,7 +3498,7 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, kick_sync_wait_bal_cb); } - } else if (rq->scx.flags & SCX_RQ_BAL_CB_PENDING) { + } else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; schedule_deferred(rq); } From 5547b250676c1924090133e6f3f30195f54cdbe1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 12:29:39 -1000 Subject: [PATCH 66/69] sched_ext: Factor out the rq lock switch of scx_dsq_move() Factor the rq lock switch on entry to scx_dsq_move() and its undo on exit out into kf_switch_rq_lock() and kf_restore_rq_lock() so that other kfuncs operating on a task from a DSQ iteration can share them. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 63 ++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b9f80fb29255..3e707b00e78b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -9745,6 +9745,40 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = { .filter = scx_kfunc_context_filter, }; +/* + * Lock @rq from a kfunc called either from ops.dispatch(), which holds the + * dispatched rq's lock, or from a context holding no rq lock. Under core + * scheduling, dispatch can run for a sibling rq, so the locked rq is not + * necessarily this CPU's. Return the rq locked on entry, NULL if none. + */ +static struct rq *kf_switch_rq_lock(struct rq *rq, unsigned long *flags) +{ + struct rq *entry_rq; + + local_irq_save(*flags); + + entry_rq = scx_locked_rq(); + if (entry_rq) { + if (entry_rq != rq) + switch_rq_lock(entry_rq, rq); + } else { + raw_spin_rq_lock(rq); + } + return entry_rq; +} + +/* undo kf_switch_rq_lock(), @rq is the rq locked now */ +static void kf_restore_rq_lock(struct rq *rq, struct rq *entry_rq, unsigned long flags) +{ + if (entry_rq) { + if (entry_rq != rq) + switch_rq_lock(rq, entry_rq); + } else { + scx_rq_lock_drop(rq); + raw_spin_rq_unlock_irqrestore(rq, flags); + } +} + static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, struct task_struct *p, u64 dsq_id, u64 enq_flags, bool priq) @@ -9779,27 +9813,8 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, if (unlikely(READ_ONCE(sch->aborting))) return false; - /* - * Can be called from either ops.dispatch() holding the dispatched rq's - * lock or any context where no rq lock is held. If latter, lock @p's - * task_rq which we'll likely need anyway. - */ src_rq = task_rq(p); - - local_irq_save(flags); - - /* - * Under core scheduling, dispatch can run for a sibling rq, so the - * locked rq is not necessarily this CPU's. - */ - locked_rq = scx_locked_rq(); - - if (locked_rq) { - if (locked_rq != src_rq) - switch_rq_lock(locked_rq, src_rq); - } else { - raw_spin_rq_lock(src_rq); - } + locked_rq = kf_switch_rq_lock(src_rq, &flags); p_rq = src_rq; raw_spin_lock(&src_dsq->lock); @@ -9838,13 +9853,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, p_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq); dispatched = true; out: - if (locked_rq) { - if (locked_rq != p_rq) - switch_rq_lock(p_rq, locked_rq); - } else { - scx_rq_lock_drop(p_rq); - raw_spin_rq_unlock_irqrestore(p_rq, flags); - } + kf_restore_rq_lock(p_rq, locked_rq, flags); kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME); From b3768d058c98d4e8114a246d3267d7dae1c123fb Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 13:09:28 -1000 Subject: [PATCH 67/69] sched_ext: Add scx_bpf_dsq_reenq_task() to re-enqueue single tasks from a DSQ iteration scx_bpf_dsq_reenq() re-enqueues a whole DSQ, but a scheduler that loses the cpus serving a DSQ, such as a sub-scheduler after a cap revoke, needs only the stranded tasks placed again. Add scx_bpf_dsq_reenq_task(), which marks a task from a DSQ iteration. When the iteration ends, the DSQ's deferred re-enqueue walk is scheduled once and re-enqueues only the marked tasks. The kfunc locks the task's rq and the DSQ like scx_bpf_dsq_move() and is callable from the same contexts. The mark is cleared when the task leaves the DSQ, so a task consumed before the walk is not re-enqueued. An iteration's DSQ can be destroyed before the iteration ends, so schedule_dsq_reenq() now skips a destroyed DSQ instead of aborting the scheduler. scx_bpf_dsq_reenq() racing scx_bpf_destroy_dsq() takes the same path. Declare the kfunc for the tools and add a compat probe. v2: Note in the kerneldoc that another iteration's re-enqueue may take a marked task early, and that a task on another scheduler returns false (Andrea). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 1 + kernel/sched/ext/ext.c | 82 +++++++++++++++++++++++- kernel/sched/ext/internal.h | 2 + tools/sched_ext/include/scx/common.bpf.h | 1 + tools/sched_ext/include/scx/compat.bpf.h | 6 ++ 5 files changed, 89 insertions(+), 3 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index ae696d11dac9..efd635a99309 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -157,6 +157,7 @@ enum scx_ent_flags { /* scx_entity.dsq_flags */ enum scx_ent_dsq_flags { SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ + SCX_TASK_DSQ_REENQ = 1 << 1, /* marked by scx_bpf_dsq_reenq_task() */ }; enum scx_dsq_lnode_flags { diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3e707b00e78b..14fe63428036 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -555,11 +555,13 @@ enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, + __SCX_DSQ_ITER_HAS_REENQ = 1U << 29, __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | + __SCX_DSQ_ITER_HAS_REENQ | __SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME, }; @@ -1292,7 +1294,9 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, WRITE_ONCE(dru->flags, dru->flags | reenq_flags); } } else { - scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); + /* destroyed since the caller resolved it, nothing to reenq */ + if (READ_ONCE(dsq->id) != SCX_DSQ_INVALID) + scx_error(sch, "DSQ 0x%llx not allowed for reenq", dsq->id); return; } @@ -1918,6 +1922,7 @@ void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq) RB_CLEAR_NODE(&p->scx.dsq_priq); p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ; } + p->scx.dsq_flags &= ~SCX_TASK_DSQ_REENQ; list_del_init(&p->scx.dsq_list.node); dsq_dec_nr(dsq, p); @@ -5095,7 +5100,9 @@ static void process_deferred_reenq_locals(struct rq *rq) static bool user_task_should_reenq(struct task_struct *p, u64 reenq_flags, u32 *reason) { *reason = SCX_TASK_REENQ_KFUNC; - return reenq_flags & SCX_REENQ_ANY; + if (reenq_flags & SCX_REENQ_ANY) + return true; + return (reenq_flags & SCX_REENQ_MARKED) && (p->scx.dsq_flags & SCX_TASK_DSQ_REENQ); } static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flags) @@ -10087,6 +10094,8 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +/* also in scx_kfunc_ids_unlocked, same contexts as scx_bpf_dsq_move() */ +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) #ifdef CONFIG_EXT_SUB_SCHED BTF_ID_FLAGS(func, scx_bpf_sub_dispatch, KF_IMPLICIT_ARGS) #endif @@ -10202,6 +10211,7 @@ BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_slice, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_set_vtime, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move, KF_RCU) BTF_ID_FLAGS(func, scx_bpf_dsq_move_vtime, KF_RCU) +BTF_ID_FLAGS(func, scx_bpf_dsq_reenq_task, KF_RCU) /* also in scx_kfunc_ids_select_cpu: also callable from ops.select_cpu()/ops.enqueue() */ BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU) BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU) @@ -10610,7 +10620,8 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator * @it: iterator to destroy * - * Undo bpf_iter_scx_dsq_new(). + * Undo bpf_iter_scx_dsq_new(). If the iteration marked tasks with + * scx_bpf_dsq_reenq_task(), schedule their re-enqueue. */ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) { @@ -10626,6 +10637,13 @@ __bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) list_del_init(&kit->cursor.node); raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); } + + if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_REENQ) { + guard(preempt)(); + schedule_dsq_reenq(kit->dsq->sched, kit->dsq, SCX_REENQ_MARKED, + scx_locked_rq()); + } + kit->dsq = NULL; } @@ -10714,6 +10732,64 @@ __bpf_kfunc void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags, schedule_dsq_reenq(sch, dsq, reenq_flags, locked_rq); } +/** + * scx_bpf_dsq_reenq_task - Re-enqueue a task from a DSQ iteration + * @it__iter: DSQ iterator in progress + * @p: task to re-enqueue + * + * Mark @p, which is on the DSQ iterated by @it__iter and was queued before the + * iteration started, for re-enqueue. When the iteration ends, the DSQ's marked + * tasks go through ops.enqueue() with %SCX_ENQ_REENQ in DSQ order and the other + * tasks on the DSQ keep their positions. A marked task that leaves the DSQ in + * the meantime is not re-enqueued. The mark belongs to @p, so a re-enqueue + * scheduled by another iteration of the DSQ may re-enqueue @p before this + * iteration ends. @p must be on the DSQ's scheduler. + * + * Re-enqueues are performed asynchronously. Can be called from ops.dispatch() + * or any BPF context which doesn't hold a rq lock (e.g. BPF timers or SYSCALL + * programs). + * + * Return %true if @p was marked, %false if @p left the DSQ or was queued after + * the iteration started, @p is on another scheduler, or the iterator holds no + * DSQ. + */ +__bpf_kfunc bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it__iter, + struct task_struct *p) +{ + struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; + struct scx_dispatch_q *dsq = kit->dsq; + struct rq *src_rq, *entry_rq; + unsigned long flags; + bool marked = false; + + /* NULL after a failed bpf_iter_scx_dsq_new(), see scx_dsq_move() */ + if (unlikely(!dsq)) + return false; + + /* the cursor check needs @p's rq locked */ + src_rq = task_rq(p); + entry_rq = kf_switch_rq_lock(src_rq, &flags); + + scoped_guard (raw_spinlock, &dsq->lock) { + if (nldsq_cursor_lost_task(&kit->cursor, src_rq, dsq, p)) + break; + + if (unlikely(!scx_task_on_sched(dsq->sched, p))) { + scx_error(dsq->sched, "scx_bpf_dsq_reenq_task() on %s[%d] but the task belongs to a different scheduler", + p->comm, p->pid); + break; + } + + p->scx.dsq_flags |= SCX_TASK_DSQ_REENQ; + /* the walk is scheduled once, by bpf_iter_scx_dsq_destroy() */ + kit->cursor.flags |= __SCX_DSQ_ITER_HAS_REENQ; + marked = true; + } + + kf_restore_rq_lock(src_rq, entry_rq, flags); + return marked; +} + /** * scx_bpf_reenqueue_local___v2 - Re-enqueue tasks on a local DSQ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b408993af794..29e69a83d876 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1946,6 +1946,8 @@ enum scx_reenq_flags { /* internal: kernel-issued on cap revoke, not accepted from BPF */ SCX_REENQ_CAP_REVOKE = 1LLU << 1, + /* internal: tasks marked by scx_bpf_dsq_reenq_task() */ + SCX_REENQ_MARKED = 1LLU << 2, __SCX_REENQ_FILTER_MASK = 0xffffLLU, diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 651bd4952647..9376d5704a47 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -75,6 +75,7 @@ struct task_struct *scx_bpf_dsq_peek(u64 dsq_id) __ksym __weak; int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak; struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak; void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak; +bool scx_bpf_dsq_reenq_task(struct bpf_iter_scx_dsq *it, struct task_struct *p) __ksym __weak; void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak; void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym; void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak; diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index c26024a6e2b1..254882bd8ce6 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -498,6 +498,12 @@ static inline bool __COMPAT_has_generic_reenq(void) return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); } +/* v7.4: Add scx_bpf_dsq_reenq_task(). */ +static inline bool __COMPAT_has_dsq_reenq_task(void) +{ + return bpf_ksym_exists(scx_bpf_dsq_reenq_task); +} + /* * v6.19: The new void variant can be called from anywhere while the older v1 * variant can only be called from ops.cpu_release(). The double ___ prefixes on From cce3618e2d4bd309966bed9930692e5d72e3a4ef Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 12:29:39 -1000 Subject: [PATCH 68/69] sched_ext: scx_qmap: Also re-enqueue stranded tasks with scx_bpf_dsq_reenq_task() scan_shared_dsq() rescues a task stranded on SHARED_DSQ or HIGHPRI_DSQ by moving it to the rescue path with scx_bpf_dsq_move(). For a task with an odd pid, mark it with scx_bpf_dsq_reenq_task() instead, which sends it back through qmap_enqueue() and its rescue, so that both kfuncs are exercised. scan_shared_dsq() runs from qmap_dispatch() and from the monitor timer, so the kfunc can be called both with the dispatch rq locked and with no rq lock held. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- tools/sched_ext/scx_qmap.bpf.c | 50 +++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 61143a2d79f8..e8e451be862d 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -651,10 +651,35 @@ static void update_core_sched_head_seq(struct task_struct *p) qa.core_sched_head_seqs[idx] = taskc->core_sched_seq; } +/* + * Send @p, stranded on the DSQ @it iterates, to the kernel rescue path. To + * exercise both kfuncs, move tasks with an even pid there with + * scx_bpf_dsq_move() and re-enqueue the others with scx_bpf_dsq_reenq_task(), + * which takes them through qmap_enqueue()'s rescue. The choice is per task so + * that a marked task isn't moved by another scan before the re-enqueue reaches + * it. + */ +static __always_inline void rescue_stranded(struct bpf_iter_scx_dsq *it, + struct task_struct *p, task_ctx_t *taskc) +{ + s32 c; + + if (p->pid & 1) { + scx_bpf_dsq_reenq_task(it, p); + return; + } + + c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); + if (c >= 0 && c < scx_bpf_nr_cids()) { + __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); + scx_bpf_dsq_move(it, p, SCX_DSQ_LOCAL_ON | c, SCX_ENQ_RESCUE); + } +} + /* * One pass over SHARED_DSQ: rescue stranded tasks and boost highpri ones. A * task whose cids were lost while it was queued in the fifos would strand on - * SHARED_DSQ, which is consumed only on self cids it can't run on - move it to + * SHARED_DSQ, which is consumed only on self cids it can't run on - send it to * the kernel rescue path. One whose cids were lost after the highpri cull is * likewise rescued out of HIGHPRI_DSQ below. * @@ -663,9 +688,9 @@ static void update_core_sched_head_seq(struct task_struct *p) * consuming them first. This makes minor difference only when dsp_batch is * larger than 1. * - * scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and - * non-rq-lock holding BPF programs. As demonstration, this function is called - * from qmap_dispatch() and monitor_timerfn(). + * scx_bpf_dsq_move[_vtime]() and scx_bpf_dsq_reenq_task() are allowed both from + * ops.dispatch() and non-rq-lock holding BPF programs. As demonstration, this + * function is called from qmap_dispatch() and monitor_timerfn(). */ static bool scan_shared_dsq(bool from_timer) { @@ -677,19 +702,13 @@ static bool scan_shared_dsq(bool from_timer) bpf_for_each(scx_dsq, p, SHARED_DSQ, 0) { static u64 highpri_seq; task_ctx_t *taskc; - s32 c; if (!(taskc = lookup_task_ctx(p))) return false; /* stranded? rescue - it can't be dispatched here either way */ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) { - c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); - if (c >= 0 && c < scx_bpf_nr_cids()) { - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - SCX_ENQ_RESCUE); - } + rescue_stranded(BPF_FOR_EACH_ITER, p, taskc); continue; } @@ -723,19 +742,12 @@ static bool scan_shared_dsq(bool from_timer) &qa.usable_cids.mask, this_cid + 1); if (cid >= nr_cids) { - s32 c; - /* self cids lack caps in effect yet, leave it queued */ if (cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) continue; /* stranded after the cull - rescue it from here */ - c = cmask_next_set_wrap(&taskc->cpus_allowed, 0); - if (c >= 0 && c < nr_cids) { - __sync_fetch_and_add(&qa.nr_rescue_dsp, 1); - scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c, - SCX_ENQ_RESCUE); - } + rescue_stranded(BPF_FOR_EACH_ITER, p, taskc); continue; } From 1f5dde349ac2fca1f119ae8de3f0d2f1c2a8d338 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 13:56:04 -1000 Subject: [PATCH 69/69] sched_ext: Drop the locks periodically while walking a user DSQ for re-enqueue reenq_user() drops dsq->lock around each re-enqueue and its rq lock every SCX_TASK_ITER_BATCH re-enqueues. A walk filtered by SCX_REENQ_MARKED skips unmarked tasks without dropping either, so a long run of unmarked tasks is walked in one hold of both locks. Count the visited tasks instead of the re-enqueued ones and drop both locks every SCX_TASK_ITER_BATCH visits. An unfiltered walk drops the locks as often as before. Suggested-by: Andrea Righi Link: https://lore.kernel.org/all/atP-LIKpxPhR-Nuc@gpd4/ Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 14fe63428036..07857039aee6 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -5111,7 +5111,7 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag struct scx_sched *sch = dsq->sched; struct scx_dsq_list_node cursor = INIT_DSQ_LIST_CURSOR(cursor, dsq, 0); struct task_struct *p; - s32 nr_enqueued = 0; + s32 nr_visited = 0; lockdep_assert_rq_held(rq); @@ -5121,6 +5121,21 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag struct rq *task_rq; u32 reason; + /* + * Count skipped tasks too so that a long walk doesn't hold the + * locks throughout. The cursor keeps the position. + */ + if (!(++nr_visited % SCX_TASK_ITER_BATCH)) { + raw_spin_unlock(&dsq->lock); + if (locked_rq) { + scx_rq_lock_drop(locked_rq); + raw_spin_rq_unlock(locked_rq); + locked_rq = NULL; + } + cpu_relax(); + raw_spin_lock(&dsq->lock); + } + p = nldsq_cursor_next_task(&cursor, dsq); if (!p) break; @@ -5157,13 +5172,6 @@ static void reenq_user(struct rq *rq, struct scx_dispatch_q *dsq, u64 reenq_flag scx_do_enqueue_task(task_rq, p, SCX_ENQ_REENQ, -1); - if (!(++nr_enqueued % SCX_TASK_ITER_BATCH)) { - scx_rq_lock_drop(locked_rq); - raw_spin_rq_unlock(locked_rq); - locked_rq = NULL; - cpu_relax(); - } - raw_spin_lock(&dsq->lock); }