From c71ecdf0d89d6871bd57560fcf40997b3e6d4d7e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 15:59:47 -1000 Subject: [PATCH 01/34] sched_ext: Factor out cid_pshard() Factor the cid to pshard lookup out of the ecaps sync paths into cid_pshard() so that new sync paths can share it. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/sub.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 1345e4174775..7edad2ddaa54 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1063,6 +1063,12 @@ static void discard_queued_syncs(struct rq *rq) init_llist_node(pos); } +/* the pshard holding @sch's caps on @cid */ +static struct scx_pshard *cid_pshard(struct scx_sched *sch, s32 cid) +{ + return sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; +} + /** * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq @@ -1079,7 +1085,7 @@ static void discard_queued_syncs(struct rq *rq) void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) { s32 cpu = cpu_of(rq); - s32 cid, shard; + s32 cid; struct llist_node *batch, *pos, *tmp; u64 lost_all = 0; @@ -1101,13 +1107,12 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) /* @cid is valid here: the cpu is active with queued syncs */ cid = __scx_cpu_to_cid(cpu); - shard = rcu_dereference_all(scx_cid_to_shard)[cid]; batch = llist_del_all(&rq->scx.ecaps_to_sync); llist_for_each_safe(pos, tmp, batch) { struct scx_sched_pcpu *pcpu = container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); - struct scx_pshard *ps = pcpu->sch->pshard[shard]; + struct scx_pshard *ps = cid_pshard(pcpu->sch, cid); u64 old, ecaps, lost, gained; init_llist_node(pos); @@ -1229,7 +1234,7 @@ void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) void scx_online_ecaps(struct rq *rq) { struct scx_sched *root, *pos; - s32 cid, shard; + s32 cid; /* * Only a live hierarchy can have ecaps to reseed. This also keeps the @@ -1244,7 +1249,6 @@ void scx_online_ecaps(struct rq *rq) root = scx_root_protected(); cid = __scx_cpu_to_cid(cpu_of(rq)); - shard = rcu_dereference_all(scx_cid_to_shard)[cid]; scx_for_each_descendant_pre(pos, root) { struct scx_pshard *ps; @@ -1253,7 +1257,7 @@ void scx_online_ecaps(struct rq *rq) if (!pos->level) continue; - ps = pos->pshard[shard]; + ps = cid_pshard(pos, cid); guard(raw_spinlock)(&ps->lock); queue_sync_ecaps(pos, cid); } From c28c637dd804462294d261bfd55ab48264932f95 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 15:59:48 -1000 Subject: [PATCH 02/34] sched_ext: Free a sub-scheduler a grace period after discarding its ecaps syncs The free path removes a sub-scheduler's ecaps sync node from each cpu's list and then cycles the rq lock until the node reads off-list, as a sync batch can hold the node privately. Cycling the lock is correct only because the sched is bypassing and a bypassing sched's sync calls no op, so the batch touches the pcpu within one rq lock hold. Tying the free path to what the sync does for a bypassing sched is unusual and brittle, and it gets in the way of changing the sync. Delivering ecaps to a bypassing sched or its parent would break the wait, as a delivery drops the rq lock. Wait for the batches with RCU instead, the usual way to wait for lockless users before a free. Discard the nodes after the first grace period as before, and free the sched after a second one. The first grace period waits for the paths that found the sched before it was unlinked, such as the sub-cap kfuncs and the cpu hotplug hooks. Until they finish, they can queue its syncs, and a cpu can take a queued node into a sync batch that the first grace period doesn't wait for. After it, nothing queues the syncs, and the discards remove the nodes still queued. A batch that took a node keeps irqs off from llist_del_all() to its last pcpu write, even across the rq lock drops of a delivery. The second grace period waits for it whether or not it called an op. The exit and disable irq_works, the helper kthread and the bypass load balancing timer are stopped ahead of the per-cpu checks as before, so they move into scx_sched_quiesce_rcu_work() with the checks. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 76 ++++++++++++++++++++++++++++++------------ kernel/sched/ext/sub.c | 51 +++++++++++----------------- 2 files changed, 74 insertions(+), 53 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 07857039aee6..255607739fb1 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6028,10 +6028,6 @@ static void scx_sched_free_rcu_work(struct work_struct *work) struct scx_dispatch_q *dsq; int cpu, node; - irq_work_sync(&sch->propagate_exit_irq_work); - irq_work_sync(&sch->disable_irq_work); - kthread_destroy_worker(sch->helper); - timer_shutdown_sync(&sch->bypass_lb_timer); free_cpumask_var(sch->bypass_lb_donee_cpumask); free_cpumask_var(sch->bypass_lb_resched_cpumask); free_cpumask_var(sch->stall_cpus); @@ -6049,22 +6045,6 @@ static void scx_sched_free_rcu_work(struct work_struct *work) for_each_possible_cpu(cpu) { struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - /* - * $sch would have entered bypass mode before the RCU grace - * period. As that blocks new deferrals, all - * deferred_reenq_local_node's must be off-list by now. - */ - WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); - - /* remove the queued ecaps sync so the pcpu can be freed */ - scx_discard_ecaps_to_sync(cpu, pcpu); - - /* - * Bypass blocks new kicks. Flush the kick irq_work so this - * pcpu's to_kick_node is off the list before it is freed. - */ - irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); - WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node)); free_cpumask_var(pcpu->cpus_to_kick); free_cpumask_var(pcpu->cpus_to_kick_if_idle); free_cpumask_var(pcpu->cpus_to_preempt); @@ -6106,11 +6086,65 @@ static void scx_sched_free_rcu_work(struct work_struct *work) kfree(sch); } +/** + * scx_sched_quiesce_rcu_work - Quiesce a sched and queue its free + * @work: &scx_sched.rcu_work + * + * $sch is freed after two RCU grace periods because of its ecaps sync nodes. + * + * The first grace period runs between scx_kobj_release() and this function. It + * waits for the paths that found $sch before scx_unlink_sched(), such as the + * sub-cap kfuncs and the cpu hotplug hooks. Until they finish, they can queue + * $sch's syncs, and a cpu can take a queued node into a sync batch. The first + * grace period doesn't wait for a batch that started after it began. + * + * Nothing queues $sch's syncs now. Remove the nodes still on the cpus' lists. A + * batch that took a node can still write $sch's pcpu, and a delivery in the + * batch can drop the rq lock before that write. The batch keeps irqs off from + * taking the node to its last pcpu write, so the second grace period waits for + * it before scx_sched_free_rcu_work() frees $sch. + */ +static void scx_sched_quiesce_rcu_work(struct work_struct *work) +{ + struct rcu_work *rcu_work = to_rcu_work(work); + struct scx_sched *sch = container_of(rcu_work, struct scx_sched, rcu_work); + s32 cpu; + + irq_work_sync(&sch->propagate_exit_irq_work); + irq_work_sync(&sch->disable_irq_work); + kthread_destroy_worker(sch->helper); + timer_shutdown_sync(&sch->bypass_lb_timer); + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + + /* + * $sch would have entered bypass mode before the first RCU + * grace period. As that blocks new deferrals, all + * deferred_reenq_local_node's must be off-list by now. + */ + WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node)); + + /* remove the queued ecaps sync so the pcpu can be freed */ + scx_discard_ecaps_to_sync(cpu, pcpu); + + /* + * Bypass blocks new kicks. Flush the kick irq_work so this + * pcpu's to_kick_node is off the list before it is freed. + */ + irq_work_sync(&cpu_rq(cpu)->scx.kick_cpus_irq_work); + WARN_ON_ONCE(!list_empty(&pcpu->to_kick_node)); + } + + INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); + queue_rcu_work(system_dfl_wq, &sch->rcu_work); +} + static void scx_kobj_release(struct kobject *kobj) { struct scx_sched *sch = container_of(kobj, struct scx_sched, kobj); - INIT_RCU_WORK(&sch->rcu_work, scx_sched_free_rcu_work); + INIT_RCU_WORK(&sch->rcu_work, scx_sched_quiesce_rcu_work); queue_rcu_work(system_dfl_wq, &sch->rcu_work); } diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 7edad2ddaa54..1923bdc33d6b 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1318,8 +1318,10 @@ static void clear_all_caps(struct scx_sched *sch) } /* - * @pcpu's sched was unhashed before the grace period, so nothing re-queues its - * sync node. Remove the node from @rq's pending list so the pcpu can be freed. + * Remove @pcpu's ecaps sync node from @cpu's pending list. The first grace + * period has passed, so nothing queues the node again. The second grace period + * waits for a sync batch that took the node earlier, see + * scx_sched_quiesce_rcu_work(). */ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) { @@ -1327,43 +1329,28 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) struct llist_node *head = NULL, *tail = NULL; struct llist_node *pos, *tmp; + guard(rq_lock_irqsave)(rq); + + if (!llist_on_list(&pcpu->ecaps_to_sync_node)) + return; + /* * llist can't unlink a single node. Take all queued nodes, drop @pcpu's * and resplice the rest. Nodes in the taken batch read as on-list * throughout, so queue_sync_ecaps() stays correct. */ - if (llist_on_list(&pcpu->ecaps_to_sync_node)) { - scoped_guard (rq_lock_irqsave, rq) { - llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) { - if (pos == &pcpu->ecaps_to_sync_node) { - init_llist_node(pos); - } else { - pos->next = head; - head = pos; - if (!tail) - tail = pos; - } - } - if (head) - llist_add_batch(head, tail, &rq->scx.ecaps_to_sync); + llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) { + if (pos == &pcpu->ecaps_to_sync_node) { + init_llist_node(pos); + } else { + pos->next = head; + head = pos; + if (!tail) + tail = pos; } } - - /* - * An in-flight scx_process_sync_ecaps() batch may still hold the node - * privately across dispatch-induced rq unlocks, reading as on-list. - * - * Because a bypassing sched gets no op call, init_llist_node() and all - * @pcpu accesses share one contiguous lock hold, off-list under the rq - * lock means @pcpu won't be accessed again. - */ - while (true) { - scoped_guard (rq_lock_irqsave, rq) { - if (!llist_on_list(&pcpu->ecaps_to_sync_node)) - return; - } - cpu_relax(); - } + if (head) + llist_add_batch(head, tail, &rq->scx.ecaps_to_sync); } /** From 47769db146d90e33107e43e12ca7fc1e2dafbc32 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 15:59:49 -1000 Subject: [PATCH 03/34] sched_ext: Report a sub-scheduler's ecaps on attach and detach A sub-scheduler learns which cids it holds through ops.sub_ecaps_updated(), but the caps granted from ops.sub_attach() reach it only after its enable bypass lifts, when it is already scheduling its tasks. Report the caps at the end of the enable, before bypass lifts. On disable, report their loss before any task goes back to the parent, so that the parent hears of the loss of every cap it was told of on an active cpu. ops.sub_detach() remains where the parent cleans up after the sub-scheduler. Deliveries no longer depend on bypass, so ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() also run while a scheduler bypasses. The replay at unbypass goes, and the enable's bypass returns to after ops.sub_attach(). A sub-scheduler's ecaps now change only with the deliveries that report them, so that whenever its ops run under the rq lock, its caps are the ones it was last told of. A sync before the attach report leaves them alone, and the detach report zeroes them as it delivers the loss. A cpu going offline still drops its caps without a report, and those are the only changes not delivered. clear_all_caps() drops that unreported loss on inactive cpus. The following things need care. A report reschedules only the cpus the sub-scheduler was ever granted a cap on. Rescheduling every cpu would disrupt a large machine split many ways, and which cpus still owe a delivery can't be read off the caps. The report and a queued sync reschedule the cpu from hard irq work on that cpu, not with a kick or a direct resched. dispatch_one() processes the syncs only at its start, and a kick only guarantees that a scheduling event completes after it, which can be the pick in progress. That pick has already processed the syncs and clears the resched, and a cpu that then idles or runs a task with an infinite slice leaves the report waiting. A pick runs with IRQs off, so the irq work runs after any pick in progress on that cpu. Under core scheduling, a sibling's committed pick can switch the cpu without dispatch_one(), and a sync queued after that pick is processed only at the cpu's next scheduling event. A cpu running a task of a higher class isn't rescheduled, as its pick would return that task again. While the root bypasses, a queued sync doesn't reschedule the cpu, as a grant or revoke can run in the PM syscore phase, where irq work can malfunction. The report never runs there and reschedules the cpu itself. The report waits on an rcuwait woken from hard irq work. Under core scheduling, a cpu going down can process a sibling's marked sync after hotplug has parked its irq_work thread. On PREEMPT_RT, a regular irq_work queued there would be left behind, and a hard irq can't take a wait queue's lock. v2: Reschedule from hard irq work on the cpu instead of a kick (sashiko). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 38 ++++- kernel/sched/ext/internal.h | 52 ++++++- kernel/sched/ext/sub.c | 272 ++++++++++++++++++++++++++---------- kernel/sched/ext/sub.h | 2 - kernel/sched/sched.h | 1 + 5 files changed, 277 insertions(+), 88 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 255607739fb1..f37a4c9d4121 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1152,6 +1152,36 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +static void resched_irq_workfn(struct irq_work *irq_work) +{ + struct rq *rq = container_of(irq_work, struct rq, scx.resched_irq_work); + + guard(rq_lock)(rq); + /* a higher class task would only be picked again, see kick_one_cpu() */ + if (!sched_class_above(rq->donor->sched_class, &ext_sched_class)) + resched_curr(rq); + scx_rq_lock_drop(rq); +} + +/** + * scx_resched_curr_deferred - Reschedule @rq's cpu from hard irq work on it + * @rq: rq to reschedule + * + * A kick, like a direct resched, only guarantees that a scheduling event on the + * cpu completes after it, and that can be the one in progress. Work that a pick + * processes only at the start of its dispatch, such as the ecaps syncs, can be + * queued after that point and miss the pick. A pick on @rq's cpu runs with IRQs + * off, so the irq work runs after it, and the scheduling event its resched + * triggers starts after this call. A sibling's core scheduling pick for @rq + * isn't covered. A cpu running a task of a higher class isn't rescheduled, as + * its pick would return that task again. @rq's cpu must stay online until the + * irq work is queued. + */ +void scx_resched_curr_deferred(struct rq *rq) +{ + irq_work_queue_on(&rq->scx.resched_irq_work, cpu_of(rq)); +} + #ifdef CONFIG_NO_HZ_FULL static void scx_proxy_tick_bal_cb(struct rq *rq) { @@ -6791,6 +6821,9 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * * - ops.dispatch() is ignored. * + * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called. + * The bypassed tasks are on the bypass DSQs, out of their reach. + * * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to * the tail of the queue. @@ -6870,10 +6903,8 @@ void scx_bypass(struct scx_sched *sch, bool bypass) pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; } else { pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; - if (was_bypassing) { + if (was_bypassing) unbypass_renotify_idle(rq, pos, pcpu); - scx_unbypass_replay_ecaps(rq, pos); - } } } @@ -9510,6 +9541,7 @@ void __init init_sched_ext_class(void) INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); + rq->scx.resched_irq_work = IRQ_WORK_INIT_HARD(resched_irq_workfn); if (cpu_online(cpu)) cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 29e69a83d876..a3c63522c38a 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -878,9 +878,14 @@ struct sched_ext_ops { * @args: argument container, see the struct definition * * The sub-scheduler holds no caps by this point and can no longer - * affect any cid. Its caps went away without a - * sub_child_ecaps_updated() report, so whatever was delegated to it, - * e.g. cpuperf targets, is the parent's to restore here. + * affect any cid. Every cap reported through + * ops.sub_child_ecaps_updated() was also reported lost there before the + * sub-scheduler's tasks came back to this sched, except on a cpu that + * went offline. Nothing is reported for a sub-scheduler whose enable + * failed before its tasks were handed over. If this sched is being + * disabled too, it can miss these reports. On a cpu whose loss wasn't + * reported, whatever was delegated, e.g. cpuperf targets, is the + * parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -914,6 +919,14 @@ struct sched_ext_ops { * change is in effect on the cpu. Runs in dispatch context with rq lock * held, and can perform all operations allowed in ops.dispatch() * including inserting/moving tasks. + * + * Deliveries start after this sched's tasks are handed to it and before + * it makes its first scheduling decision. The caps granted from + * ops.sub_attach() and any change since are reported then. A cpu that + * isn't online at that point reports them once it comes online. + * Deliveries continue while this sched bypasses. Its tasks are then on + * the bypass DSQs, where this op can't insert or move them. Deliveries + * stop once this sched is being disabled. */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); @@ -940,11 +953,16 @@ struct sched_ext_ops { * @before: the child's effective caps as of the last delivery * @after: the child's effective caps now * - * Invoked right after the child's sub_ecaps_updated(), once a grant or - * revoke is in effect on the cpu. From here on the parent can act on a + * Invoked right after the child's ops.sub_ecaps_updated(), once a + * change is in effect on the cpu. From here on the parent can act on a * cap the child lost, e.g. reset the cpuperf target or schedule on the - * cid. A detaching child's caps go away without a report, see - * sub_detach(). A cpu going offline drops caps without a report. + * cid. A disabling child gets no ops.sub_ecaps_updated(), so only this + * op is invoked for its loss. A cpu going offline drops caps without a + * report. + * + * Deliveries start as for the child's ops.sub_ecaps_updated(), continue + * while either sched bypasses and stop once this sched is being + * disabled. * * Runs in dispatch context with rq lock held, and can perform all * operations allowed in ops.dispatch() including inserting/moving @@ -1414,6 +1432,10 @@ struct scx_sched; enum scx_sched_pcpu_flags { SCX_SCHED_PCPU_BYPASSING = 1LLU << 0, + /* ecaps deliveries are enabled, see ecaps_enable() */ + SCX_SCHED_PCPU_ECAPS_ENABLED = 1LLU << 1, + /* a report is pending, see ecaps_report() */ + SCX_SCHED_PCPU_ECAPS_REPORT = 1LLU << 2, }; /* dispatch buf */ @@ -1474,6 +1496,16 @@ struct scx_sched_pcpu { struct llist_node ecaps_to_sync_node; /* owed a forced update_idle() re-notify on this cpu */ bool idle_renotify; + /* + * The sched was granted a cap on this cpu at some point. ecaps_report() + * reschedules only these cpus, as rescheduling every cpu on each + * sub-sched enable and disable would disrupt a large machine split many + * ways. Exactly which cpus owe a delivery is hard to track, as + * deliveries are asynchronous and reading caps[] and ecaps doesn't + * tell. A cpu granted a cap at some point is a simple superset. Written + * under pshard->lock. + */ + bool had_caps; /* * ecaps as of the last sub_ecaps_updated() and * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps(). @@ -1751,6 +1783,11 @@ struct scx_sched { bool linked; /* on ->children, see scx_link_sched() */ bool sub_attached; + /* ecaps_enable() has run, under scx_enable_mutex */ + bool ecaps_enabled; + + /* marked cpus whose sync is pending, see ecaps_report() */ + atomic_t nr_ecaps_reports; #endif /* CONFIG_EXT_SUB_SCHED */ /* @@ -2260,6 +2297,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch); __printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...); void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +void scx_resched_curr_deferred(struct rq *rq); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 1923bdc33d6b..a3f7c0be98ac 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -42,6 +42,17 @@ static s64 scx_rescue_sat_delta_ns; static unsigned long scx_rescue_decay_halflife; static unsigned long scx_rescue_overload_after; +/* + * ecaps_report() waits here for its marked syncs. A cpu going down keeps + * picking tasks after hotplug parks its irq_work thread. Its picks can process + * a marked sync, e.g. a core scheduling sibling's. A regular irq_work queued + * there is left behind on PREEMPT_RT, so the wakeup is hard irq work. A hard + * irq can't take a wait queue's lock, which sleeps on PREEMPT_RT. + * scx_enable_mutex serializes the reports, so the single waiter sleeps on an + * rcuwait. + */ +static struct rcuwait ecaps_report_wait = __RCUWAIT_INITIALIZER(ecaps_report_wait); + /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk * @pos: current position @@ -964,7 +975,7 @@ static void caps_updated_deliver(struct list_head *to_deliver) * so !has_op means the op is absent or the sched isn't * live yet - e.g. caps grant from ops.sub_attach(). * Either way don't consume - leave for - * scx_sub_seed_caps() to deliver once live. + * scx_sub_seed_caps() to deliver once has_op is set. */ scoped_guard (raw_spinlock, &cu->lock) { if (cu->caps && SCX_HAS_OP(sch, sub_caps_updated) && @@ -992,8 +1003,8 @@ static void caps_updated_deliver(struct list_head *to_deliver) /* * Deliver caps owed to @sch that couldn't be delivered earlier (e.g. a grant - * taken during its sub_attach(), before has_op was set). Called once @sch is - * enabled. + * taken during its sub_attach(), before has_op was set). Called once has_op is + * set. */ static void scx_sub_seed_caps(struct scx_sched *sch) { @@ -1030,8 +1041,12 @@ static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid) * @sch: sched to update * @cid: cid to update * - * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in - * dispatch_one(). + * Queue an ecaps update for @sch's @cid and reschedule the cpu so that it syncs + * in dispatch_one(). The cpu isn't rescheduled while it is offline or the root + * bypasses. Its next scheduling event processes the sync then, and a cpu coming + * back syncs again in the reseed. A grant or revoke can run in the PM syscore + * phase, where irq work can malfunction, see scx_kick_cpu(), and the root + * bypasses throughout PM. ecaps_report() reschedules the cpu itself. */ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) { @@ -1048,8 +1063,15 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) /* @cid's pshard->lock excludes concurrent queueing attempts */ if (llist_on_list(&pcpu->ecaps_to_sync_node)) return; - if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync)) - scx_kick_cpu(sch->ancestors[0], cpu, 0); + + /* + * The irq work needs an online cpu. An online cpu stays online here, as + * @cid's pshard->lock keeps preemption off, which holds off the + * stop_machine() that takes a cpu offline. + */ + if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync) && + cpu_online(cpu) && !scx_bypassing(sch->ancestors[0], smp_processor_id())) + scx_resched_curr_deferred(cpu_rq(cpu)); } /* discard @rq's queued ecaps syncs */ @@ -1069,14 +1091,31 @@ static struct scx_pshard *cid_pshard(struct scx_sched *sch, s32 cid) return sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; } +static void ecaps_report_irq_workfn(struct irq_work *irq_work) +{ + rcuwait_wake_up(&ecaps_report_wait); +} + +static struct irq_work ecaps_report_irq_work = IRQ_WORK_INIT_HARD(ecaps_report_irq_workfn); + +/* + * Complete @pcpu's report mark. The waiter in ecaps_report() can't be woken + * under the rq lock, so the last completion wakes it through irq_work. + */ +static void ecaps_report_done(struct scx_sched_pcpu *pcpu) +{ + pcpu->flags &= ~SCX_SCHED_PCPU_ECAPS_REPORT; + if (atomic_dec_and_test(&pcpu->sch->nr_ecaps_reports)) + irq_work_queue(&ecaps_report_irq_work); +} + /** * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch * * pshard->caps[] is the target configuration. pcpu->ecaps is the effective - * transposed copy owned by the cid's cpu and written only here under @rq's - * lock. + * transposed copy owned by the cid's cpu and written under @rq's lock. * * A sched that newly gains baseline access here is owed an update_idle() so it * learns the cid's idle state. Such a gain arms the per-rq @@ -1113,6 +1152,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) struct scx_sched_pcpu *pcpu = container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); struct scx_pshard *ps = cid_pshard(pcpu->sch, cid); + bool report = pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT; u64 old, ecaps, lost, gained; init_llist_node(pos); @@ -1120,8 +1160,17 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) /* pairs with smp_mb() in queue_sync_ecaps(), see there */ smp_mb(); + /* + * Before the sched's has_op is set, only the parent's op would + * run, and the change would never reach the sched. + * ecaps_enable() enables deliveries on each cpu once has_op is + * set. Until then, a sync leaves the ecaps alone. + */ old = READ_ONCE(pcpu->ecaps); - ecaps = calc_effective_caps(ps, cid); + if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) + ecaps = calc_effective_caps(ps, cid); + else + ecaps = old; WRITE_ONCE(pcpu->ecaps, ecaps); lost = old & ~ecaps; @@ -1139,25 +1188,28 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * up there. The rq recorded in it keeps the dispatches * targeting @rq. * - * Bypass propagates down the hierarchy, so a sched that isn't - * bypassing has no bypassing parent. The child's bypass state - * gates both deliveries. Both report the same before value, so - * one reported_ecaps covers them. + * Both report the same before value, so one reported_ecaps + * covers them. */ - if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) { + if (ecaps != pcpu->reported_ecaps && + (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED)) { struct scx_sched *parent = scx_parent(pcpu->sch); struct scx_dsp_ctx *dspc; /* stash @prev so nested dispatches can access it */ rq->scx.sub_dispatch_prev = prev; - if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) { + + /* a disabling sched's ops are not called */ + if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && + likely(!READ_ONCE(pcpu->sch->aborting))) { dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; dspc->rq = rq; SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps); scx_flush_dispatch_buf(pcpu->sch, rq); } - if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) { + if (SCX_HAS_OP(parent, sub_child_ecaps_updated) && + likely(!READ_ONCE(parent->aborting))) { dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx; dspc->rq = rq; SCX_CALL_OP(parent, sub_child_ecaps_updated, rq, @@ -1169,6 +1221,9 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) pcpu->reported_ecaps = ecaps; } + if (report) + ecaps_report_done(pcpu); + /* * Gaining baseline access owes an update_idle() so the sched * learns the cpu's idle state. Arm the per-rq gate so the next @@ -1191,41 +1246,6 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE); } -/** - * scx_unbypass_replay_ecaps - Replay a bypass-suppressed ecaps notification - * @rq: rq of the cpu leaving bypass - * @sch: scheduler that just left bypass on @rq's cpu - * - * scx_process_sync_ecaps() consumes syncs while bypassing without delivering - * ops.sub_ecaps_updated(), leaving reported_ecaps stale. Nothing re-queues a - * sync when bypass lifts, so without a replay a cid that never changes again - * would never be notified. The attach-time initial grants are the acute case - * as they are consumed during the enable bypass window. Re-queue a sync for - * any undelivered delta so the next dispatch delivers it. - */ -void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) -{ - s32 cpu = cpu_of(rq); - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - struct scx_pshard *ps; - s32 cid; - - lockdep_assert_rq_held(rq); - - /* root holds every cap and never uses ecaps */ - if (!sch->level) - return; - - if (READ_ONCE(pcpu->ecaps) == pcpu->reported_ecaps) - return; - - cid = __scx_cpu_to_cid(cpu); - ps = sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; - - guard(raw_spinlock)(&ps->lock); - queue_sync_ecaps(sch, cid); -} - /* * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated() @@ -1281,24 +1301,32 @@ void scx_offline_ecaps(struct rq *rq) root = scx_root_protected(); scx_for_each_descendant_pre(pos, root) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + /* root holds every cap and never uses ecaps */ if (!pos->level) continue; - WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0); + WRITE_ONCE(pcpu->ecaps, 0); + + /* deliveries have stopped on this cpu, clear its mark */ + if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT) + ecaps_report_done(pcpu); } } /* - * Clear every cap @sch holds. The pshard caps go first as they are the source a - * pending sync recomputes ecaps from. ecaps are then zeroed directly for the - * cap checks. + * Clear every cap @sch holds in the pshards. The report that follows delivers + * the loss and zeroes the ecaps. The caller holds off cpu hotplug across this + * and the report. */ static void clear_all_caps(struct scx_sched *sch) { s32 si, cpu; u32 cap_bit; + lockdep_assert_cpus_held(); + /* enable may have failed before scx_alloc_pshards() */ if (!sch->pshard) return; @@ -1312,8 +1340,13 @@ static void clear_all_caps(struct scx_sched *sch) } for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + guard(rq_lock_irqsave)(cpu_rq(cpu)); - WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0); + + /* drop an unreported loss, as the report skips inactive cpus */ + if (!cpu_active(cpu)) + pcpu->reported_ecaps = 0; } } @@ -1331,6 +1364,9 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) guard(rq_lock_irqsave)(rq); + /* ecaps_report() waited for every mark it set */ + WARN_ON_ONCE(pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT); + if (!llist_on_list(&pcpu->ecaps_to_sync_node)) return; @@ -1657,6 +1693,75 @@ static inline s32 scx_cgroup_claim_subtree(struct scx_sched *sch) { return 0; } static inline void scx_cgroup_return_subtree(struct scx_sched *sch) {} #endif +/* + * Enable @sch's ecaps deliveries on every possible cpu, so that a cpu inactive + * during the attach report delivers from the reseed at its activation. + */ +static void ecaps_enable(struct scx_sched *sch) +{ + s32 cpu; + + for_each_possible_cpu(cpu) { + guard(rq_lock_irqsave)(cpu_rq(cpu)); + per_cpu_ptr(sch->pcpu, cpu)->flags |= SCX_SCHED_PCPU_ECAPS_ENABLED; + } + sch->ecaps_enabled = true; +} + +/** + * ecaps_report - Report @sch's ecaps and wait + * @sch: sub-scheduler to report + * + * Mark every cpu @sch was ever granted a cap on, deliver its pending change + * through the dispatch path and wait for it. An inactive cpu is skipped. A cpu + * going down meanwhile has its mark cleared by scx_offline_ecaps(). + * + * Called with scx_enable_mutex held. A sched's attach and detach reports must + * not overlap, as the mark is one bit per pcpu. + */ +static void ecaps_report(struct scx_sched *sch) +{ + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + struct rq *rq = cpu_rq(cpu); + struct scx_pshard *ps; + s32 cid; + + guard(rq_lock_irqsave)(rq); + + /* syncs on an inactive cpu are discarded */ + if (!cpu_active(cpu)) + continue; + + cid = __scx_cpu_to_cid(cpu); + ps = cid_pshard(sch, cid); + guard(raw_spinlock)(&ps->lock); + + if (!pcpu->had_caps) + continue; + + pcpu->flags |= SCX_SCHED_PCPU_ECAPS_REPORT; + atomic_inc(&sch->nr_ecaps_reports); + queue_sync_ecaps(sch, cid); + + /* + * The report needs an ext dispatch on this cpu. A higher class + * task can hold the cpu, so also start the ext server, which + * picks the ext class within its period. + */ + update_rq_clock(rq); + dl_server_start(&rq->ext_server); + scx_resched_curr_deferred(rq); + } + + rcuwait_wait_event(&ecaps_report_wait, !atomic_read(&sch->nr_ecaps_reports), + TASK_UNINTERRUPTIBLE); +} + void scx_sub_disable(struct scx_sched *sch) { struct scx_sched *parent = scx_parent(sch); @@ -1678,6 +1783,24 @@ void scx_sub_disable(struct scx_sched *sch) * disable ops. */ mutex_lock(&scx_enable_mutex); + + /* + * Clear @sch's caps and report the loss before any task goes back to + * the parent. A disabling sched is granted nothing, so the caps stay + * cleared. Hold off cpu hotplug across both. The report covers every + * active cpu and the clear drops the unreported loss on the rest, so + * that no loss reaches the parent after ops.sub_detach(). + * + * An enable that failed before its attach report has nothing to report. + * If it failed before scx_link_sched(), the root's disable doesn't wait + * for it and may have retired the cid tables that the report reads. + */ + scoped_guard(cpus_read_lock) { + clear_all_caps(sch); + if (sch->ecaps_enabled) + ecaps_report(sch); + } + percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); @@ -1785,12 +1908,6 @@ void scx_sub_disable(struct scx_sched *sch) scx_unlink_sched(sch); - /* - * The parent restores what it delegated in ops.sub_detach(). @sch must - * hold no caps by then. - */ - clear_all_caps(sch); - mutex_unlock(&scx_enable_mutex); /* @@ -1950,15 +2067,6 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; - /* - * Bypass before @sch is linked and grants can reach it. The parent's - * delivery advances reported_ecaps, so a sync consumed before @sch's - * ops are registered would reach the parent and never be replayed to - * @sch. While bypassed, the syncs are consumed without a delivery and - * replayed to both at unbypass. - */ - scx_bypass(sch, true); - ret = scx_link_sched(sch); if (ret) goto err_disable; @@ -2004,6 +2112,8 @@ void scx_sub_enable_workfn(struct kthread_work *work) } sch->sub_attached = true; + scx_bypass(sch, true); + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); @@ -2145,10 +2255,16 @@ void scx_sub_enable_workfn(struct kthread_work *work) scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); - scx_bypass(sch, false); - - /* @sch is enabled; deliver any caps owed since its sub_attach() */ + /* + * Report the caps granted from ops.sub_attach() before bypass lifts and + * @sch makes its first scheduling decision. The report sleeps until its + * syncs are processed, so it runs outside the fork rwsem. + */ scx_sub_seed_caps(sch); + ecaps_enable(sch); + ecaps_report(sch); + + scx_bypass(sch, false); pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); kobject_uevent(&sch->kobj, KOBJ_ADD); @@ -2578,8 +2694,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, * the cid gains baseline access, so the holder * learns of an already-idle cid. */ - scx_cmask_for_each_cid(cid, changed_cids) + scx_cmask_for_each_cid(cid, changed_cids) { + s32 cpu = __scx_cid_to_cpu(cid); + + per_cpu_ptr(child->pcpu, cpu)->had_caps = true; queue_sync_ecaps(child, cid); + } } } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index b76d750463f0..63769205c013 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -32,7 +32,6 @@ void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); -void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); @@ -110,7 +109,6 @@ static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } static inline void scx_init_root_caps(struct scx_sched *sch) {} static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} -static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {} static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 37d4808d7f19..22a2628a04a2 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -855,6 +855,7 @@ struct scx_rq { #endif struct irq_work deferred_irq_work; struct irq_work kick_cpus_irq_work; + struct irq_work resched_irq_work; }; #endif /* CONFIG_SCHED_CLASS_EXT */ From f89d11642c1b573356382f031e3e5eb4fe87166a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 15:59:50 -1000 Subject: [PATCH 04/34] sched_ext: Deliver ecaps only while the root has the cpu online A sub-scheduler's ecaps ops run in dispatch context, and ops.cpu_online() and ops.cpu_offline() document that the cpu doesn't call ops.dispatch() outside them. The reseed of a cpu coming back is queued before the root's ops.cpu_online(), so the ops of a sub-scheduler and its parent can run for a cpu the root still considers offline. Add SCX_RQ_OPS_ONLINE, set while the root considers the cpu online, from its ops.cpu_online() or its enable until its ops.cpu_offline(). A sync outside that span leaves the ecaps alone, and the reseed, now queued after ops.cpu_online(), reports the net change. SCX_RQ_ONLINE can't be used, as the core changes it ahead of the root's view on both edges. A cpu going offline still drops its caps without a report. While at it, fix the SCX_RQ_ONLINE comments, which claimed that the flag tracks ops.cpu_online() and ops.cpu_offline(). v2: Fix the SCX_RQ_ONLINE comments. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 28 +++++++++++------- kernel/sched/ext/internal.h | 6 ++++ kernel/sched/ext/sub.c | 57 ++++++++++++++++++++++++++++++------- kernel/sched/ext/sub.h | 2 ++ kernel/sched/sched.h | 15 ++++++++-- 5 files changed, 84 insertions(+), 24 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f37a4c9d4121..02276b5b374b 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2178,11 +2178,11 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, bool scx_rq_online(struct rq *rq) { /* - * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE indicates - * the online state as seen from the BPF scheduler. cpu_active() test - * guarantees that, if this function returns %true, %SCX_RQ_ONLINE will - * stay set until the current scheduling operation is complete even if - * we aren't locking @rq. + * Test both cpu_active() and %SCX_RQ_ONLINE. %SCX_RQ_ONLINE follows the + * core's rq online state. cpu_active() test guarantees that, if this + * function returns %true, %SCX_RQ_ONLINE will stay set until the + * current scheduling operation is complete even if we aren't locking + * @rq. */ return likely((rq->scx.flags & SCX_RQ_ONLINE) && cpu_active(cpu_of(rq))); } @@ -4182,11 +4182,6 @@ static void handle_hotplug(struct rq *rq, bool online) if (scx_enabled()) scx_idle_update_selcpu_topology(&sch->ops); - if (online) - scx_online_ecaps(rq); - else - scx_offline_ecaps(rq); - /* * The tables can't be retired while this function is running as the * retirement is inside cpus_read_lock. However, scx_cpu_arg() is @@ -4215,6 +4210,14 @@ static void handle_hotplug(struct rq *rq, bool online) } } + /* + * ecaps are delivered for @cpu only after ops.cpu_online() and before + * ops.cpu_offline(). The ecaps ops may read online_cmask, so + * scx_offline_ecaps() and scx_online_ecaps() run after its update. + */ + if (!online) + scx_offline_ecaps(rq); + if (online && SCX_HAS_OP(sch, cpu_online)) SCX_CALL_OP(sch, cpu_online, NULL, cpu_or_cid); else if (!online && SCX_HAS_OP(sch, cpu_offline)) @@ -4224,6 +4227,9 @@ static void handle_hotplug(struct rq *rq, bool online) SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG, "cpu %d going %s, exiting scheduler", cpu, online ? "online" : "offline"); + + if (online) + scx_online_ecaps(rq); } void scx_rq_activate(struct rq *rq) @@ -8368,6 +8374,8 @@ static void scx_root_enable_workfn(struct kthread_work *work) goto err_disable; } + scx_init_rq_ops_online(); + /* * Make the scheduler instance visible. Must be inside cpus_read_lock(). * See handle_hotplug(). diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index a3c63522c38a..6be0146b575d 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -927,6 +927,10 @@ struct sched_ext_ops { * Deliveries continue while this sched bypasses. Its tasks are then on * the bypass DSQs, where this op can't insert or move them. Deliveries * stop once this sched is being disabled. + * + * Deliveries for a cid happen only while the root considers the cid's + * cpu online, never before the root's ops.cpu_online() or after its + * ops.cpu_offline() for the cpu. */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); @@ -980,6 +984,7 @@ struct sched_ext_ops { * * @cpu just came online. @cpu will not call ops.enqueue() or * ops.dispatch(), nor run tasks associated with other CPUs beforehand. + * ops.sub_child_ecaps_updated() runs for @cpu only after this op. */ void (*cpu_online)(s32 cpu); @@ -989,6 +994,7 @@ struct sched_ext_ops { * * @cpu is going offline. @cpu will not call ops.enqueue() or * ops.dispatch(), nor run tasks associated with other CPUs afterwards. + * ops.sub_child_ecaps_updated() runs for @cpu only before this op. */ void (*cpu_offline)(s32 cpu); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a3f7c0be98ac..2c5853d193d7 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1164,13 +1164,19 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * Before the sched's has_op is set, only the parent's op would * run, and the change would never reach the sched. * ecaps_enable() enables deliveries on each cpu once has_op is - * set. Until then, a sync leaves the ecaps alone. + * set. Deliveries also need the cpu online to the root, see + * handle_hotplug(). + * + * A sync that can't deliver leaves the ecaps alone. It may + * still carry a mark that is cleared below. Don't skip the rest + * of the loop for it. */ old = READ_ONCE(pcpu->ecaps); - if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) - ecaps = calc_effective_caps(ps, cid); - else + if (!(pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) || + !(rq->scx.flags & SCX_RQ_OPS_ONLINE)) ecaps = old; + else + ecaps = calc_effective_caps(ps, cid); WRITE_ONCE(pcpu->ecaps, ecaps); lost = old & ~ecaps; @@ -1192,7 +1198,8 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * covers them. */ if (ecaps != pcpu->reported_ecaps && - (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED)) { + (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) && + (rq->scx.flags & SCX_RQ_OPS_ONLINE)) { struct scx_sched *parent = scx_parent(pcpu->sch); struct scx_dsp_ctx *dspc; @@ -1247,15 +1254,42 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) } /* - * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync - * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated() - * only on a real change since offline. + * Called with cpus_read_lock() held before the root is published. The cpus + * active now are online to the root without ops.cpu_online(), so they can + * deliver ecaps. handle_hotplug() maintains %SCX_RQ_OPS_ONLINE from here on. + */ +void scx_init_rq_ops_online(void) +{ + s32 cpu; + + lockdep_assert_cpus_held(); + + for_each_possible_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + guard(rq_lock_irqsave)(rq); + if (cpu_active(cpu)) + rq->scx.flags |= SCX_RQ_OPS_ONLINE; + else + rq->scx.flags &= ~SCX_RQ_OPS_ONLINE; + } +} + +/* + * A cpu came back and the root's ops.cpu_online() has run. Let the cpu deliver + * ecaps again and reseed each sub-sched's ecaps on its cid. The sync recomputes + * effective caps from the pshard and fires ops.sub_ecaps_updated() only on a + * real change since offline. */ void scx_online_ecaps(struct rq *rq) { struct scx_sched *root, *pos; s32 cid; + guard(rq_lock_irqsave)(rq); + + rq->scx.flags |= SCX_RQ_OPS_ONLINE; + /* * Only a live hierarchy can have ecaps to reseed. This also keeps the * table reads below away from an enable that failed before publishing @@ -1265,8 +1299,6 @@ void scx_online_ecaps(struct rq *rq) if (!scx_enabled()) return; - guard(rq_lock_irqsave)(rq); - root = scx_root_protected(); cid = __scx_cpu_to_cid(cpu_of(rq)); @@ -1284,7 +1316,8 @@ void scx_online_ecaps(struct rq *rq) } /* - * A cpu is going down. Zero each sub-sched's in-effect ecaps so cap checks + * A cpu is going down, and the root's ops.cpu_offline() follows. Stop the cpu's + * ecaps deliveries and zero each sub-sched's in-effect ecaps so cap checks * treat the cpu as capless while offline. Pending and late-queued syncs are * discarded at consumption by scx_process_sync_ecaps() while the cpu is * inactive. Leave reported_ecaps. Ownership is unchanged, so the @@ -1313,6 +1346,8 @@ void scx_offline_ecaps(struct rq *rq) if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT) ecaps_report_done(pcpu); } + + rq->scx.flags &= ~SCX_RQ_OPS_ONLINE; } /* diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 63769205c013..46c7e2a578d1 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -32,6 +32,7 @@ void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void scx_init_rq_ops_online(void); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); @@ -109,6 +110,7 @@ static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } static inline void scx_init_root_caps(struct scx_sched *sch) {} static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} +static inline void scx_init_rq_ops_online(void) {} static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 22a2628a04a2..05cf0f0056e9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -781,12 +781,21 @@ struct cfs_rq { /* scx_rq->flags, protected by the rq lock */ enum scx_rq_flags { /* - * A hotplugged CPU starts scheduling before rq_online_scx(). Track - * ops.cpu_on/offline() state so that ops.enqueue/dispatch() are called - * only while the BPF scheduler considers the CPU to be online. + * Set while the core has the rq online, see rq_online_scx() and + * rq_offline_scx(). A hotplugged cpu starts scheduling before + * rq_online_scx(). ops.enqueue() and ops.dispatch() are called for the + * cpu only while it is set, see scx_rq_online() and + * scx_dispatch_sched(). */ SCX_RQ_ONLINE = 1 << 0, SCX_RQ_CAN_STOP_TICK = 1 << 1, + /* + * Set while the root considers the cpu online, from its + * ops.cpu_online() until its ops.cpu_offline(). ecaps are delivered + * only while it is set. %SCX_RQ_ONLINE follows the core's rq state + * instead, which changes ahead of the root's view on both edges. + */ + SCX_RQ_OPS_ONLINE = 1 << 2, SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */ SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */ SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */ From 51d2730ec19a72f0c00a858f98d98658f3063c82 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 22:22:37 -1000 Subject: [PATCH 05/34] sched_ext: Report the ecaps loss when a cpu goes offline When a cpu goes offline, a sub-scheduler's effective caps on its cid are cleared, but neither the sub-scheduler nor its parent is notified. The sub-scheduler keeps treating the cid as usable, and the parent can't act on the loss, e.g. reset the cpuperf target. Report the loss before the root's ops.cpu_offline() and the regain after its ops.cpu_online(). The zeroing at offline was the last change to a sched's ecaps outside a delivery, so they now change only with one. Syncs on an inactive cpu are processed instead of discarded, and the reseed reschedules the cpu itself, as they got no resched when queued. Suspend and hibernation now report their cpu downs and ups as well. This is temporary, and a later patch leaves them out. Two things need care. A cpu going down marks a sync for each sub-scheduler, and its hotplug thread waits until each marked sync has been processed. The cpu's own ext picks process them, and the ext server is started in case a fair task holds the cpu. Only the marked syncs count, so a flood of changes on the cid can't hold up the hotplug. The detach report reaches only active cpus. It holds off cpu hotplug, so a cpu it skips has reported its loss as it went down, and no unreported loss is left to drop. Drop the cpuperf reset from scx_qmap's ops.sub_detach(), as its ops.sub_child_ecaps_updated() now resets the targets on the loss reports. v2: Reschedule from the irq work and document the regain order. v3: Pack nr_ecaps_offline with kick_sync_pending to avoid a hole (Andrea). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/internal.h | 34 +++-- kernel/sched/ext/sub.c | 245 ++++++++++++++++++++------------- kernel/sched/sched.h | 3 +- tools/sched_ext/scx_qmap.bpf.c | 25 +--- 4 files changed, 174 insertions(+), 133 deletions(-) diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 6be0146b575d..4b9656848085 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -880,12 +880,11 @@ struct sched_ext_ops { * The sub-scheduler holds no caps by this point and can no longer * affect any cid. Every cap reported through * ops.sub_child_ecaps_updated() was also reported lost there before the - * sub-scheduler's tasks came back to this sched, except on a cpu that - * went offline. Nothing is reported for a sub-scheduler whose enable - * failed before its tasks were handed over. If this sched is being - * disabled too, it can miss these reports. On a cpu whose loss wasn't - * reported, whatever was delegated, e.g. cpuperf targets, is the - * parent's to restore here. + * sub-scheduler's tasks came back to this sched. Nothing is reported + * for a sub-scheduler whose enable failed before its tasks were handed + * over. If this sched is being disabled too, it can miss these reports. + * On a cpu whose loss wasn't reported, whatever was delegated, e.g. + * cpuperf targets, is the parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -928,9 +927,14 @@ struct sched_ext_ops { * the bypass DSQs, where this op can't insert or move them. Deliveries * stop once this sched is being disabled. * + * Until this sched is being disabled, whenever an op of this sched runs + * under a cid's rq lock, its effective caps on the cid are the @after + * of the last invocation for the cid. + * * Deliveries for a cid happen only while the root considers the cid's * cpu online, never before the root's ops.cpu_online() or after its - * ops.cpu_offline() for the cpu. + * ops.cpu_offline() for the cpu. A cpu going offline reports the loss + * of every cap on it, and a cpu coming back reports the regain. */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); @@ -961,12 +965,15 @@ struct sched_ext_ops { * change is in effect on the cpu. From here on the parent can act on a * cap the child lost, e.g. reset the cpuperf target or schedule on the * cid. A disabling child gets no ops.sub_ecaps_updated(), so only this - * op is invoked for its loss. A cpu going offline drops caps without a - * report. + * op is invoked for its loss. A parent other than the root loses its + * own caps on the cid too when the cpu goes offline, possibly before + * this op is invoked, and regains them when the cpu comes back, + * possibly after. It can't act on the cid while it holds no caps there. * * Deliveries start as for the child's ops.sub_ecaps_updated(), continue * while either sched bypasses and stop once this sched is being - * disabled. + * disabled. A cpu going offline or coming back is reported as for the + * child's ops.sub_ecaps_updated(). * * Runs in dispatch context with rq lock held, and can perform all * operations allowed in ops.dispatch() including inserting/moving @@ -1442,6 +1449,8 @@ enum scx_sched_pcpu_flags { SCX_SCHED_PCPU_ECAPS_ENABLED = 1LLU << 1, /* a report is pending, see ecaps_report() */ SCX_SCHED_PCPU_ECAPS_REPORT = 1LLU << 2, + /* a cpu going down waits for this sync, see scx_offline_ecaps() */ + SCX_SCHED_PCPU_ECAPS_OFFLINE = 1LLU << 3, }; /* dispatch buf */ @@ -1512,11 +1521,6 @@ struct scx_sched_pcpu { * under pshard->lock. */ bool had_caps; - /* - * ecaps as of the last sub_ecaps_updated() and - * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps(). - */ - u64 reported_ecaps; /* * Decaying rescue runtime consumed on this cpu, see diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 2c5853d193d7..d88a436f8963 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -53,6 +53,9 @@ static unsigned long scx_rescue_overload_after; */ static struct rcuwait ecaps_report_wait = __RCUWAIT_INITIALIZER(ecaps_report_wait); +/* a cpu going down waits here for its marked syncs, see scx_offline_ecaps() */ +static DECLARE_WAIT_QUEUE_HEAD(ecaps_offline_waitq); + /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk * @pos: current position @@ -1042,11 +1045,12 @@ static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid) * @cid: cid to update * * Queue an ecaps update for @sch's @cid and reschedule the cpu so that it syncs - * in dispatch_one(). The cpu isn't rescheduled while it is offline or the root - * bypasses. Its next scheduling event processes the sync then, and a cpu coming - * back syncs again in the reseed. A grant or revoke can run in the PM syscore - * phase, where irq work can malfunction, see scx_kick_cpu(), and the root - * bypasses throughout PM. ecaps_report() reschedules the cpu itself. + * in dispatch_one(). The cpu isn't rescheduled while it is inactive or the root + * bypasses, and its next scheduling event processes the sync then. A grant or + * revoke can run in the PM syscore phase, where irq work can malfunction, see + * scx_kick_cpu(), and the root bypasses throughout PM. Callers that need the + * sync processed promptly reschedule the cpu themselves, see + * resched_sync_ecaps(). */ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) { @@ -1054,9 +1058,10 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); /* - * Pairs with smp_mb() in __scx_process_sync_ecaps(). Either the check - * below sees the node off the list and queues it, or the in-flight sync - * sees the caps[] update made before this call. + * Pairs with smp_mb() in __scx_process_sync_ecaps(). The caller has + * written the state a sync reads, caps[] or cpu_active(). Either the + * check below finds the node off the list and queues a sync, or a sync + * in flight is still ahead of its reads and sees the new state. */ smp_mb(); @@ -1065,26 +1070,18 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) return; /* - * The irq work needs an online cpu. An online cpu stays online here, as - * @cid's pshard->lock keeps preemption off, which holds off the - * stop_machine() that takes a cpu offline. + * A cpu going down processes its syncs in its own picks, see + * scx_offline_ecaps(), and a resched for every change on the cid would + * only keep preempting its hotplug thread. The irq work needs an online + * cpu. An active cpu stays online here, as @cid's pshard->lock keeps + * preemption off, which holds off the stop_machine() that takes a cpu + * offline. */ if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync) && - cpu_online(cpu) && !scx_bypassing(sch->ancestors[0], smp_processor_id())) + cpu_active(cpu) && !scx_bypassing(sch->ancestors[0], smp_processor_id())) scx_resched_curr_deferred(cpu_rq(cpu)); } -/* discard @rq's queued ecaps syncs */ -static void discard_queued_syncs(struct rq *rq) -{ - struct llist_node *pos, *tmp; - - lockdep_assert_rq_held(rq); - - llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) - init_llist_node(pos); -} - /* the pshard holding @sch's caps on @cid */ static struct scx_pshard *cid_pshard(struct scx_sched *sch, s32 cid) { @@ -1109,13 +1106,43 @@ static void ecaps_report_done(struct scx_sched_pcpu *pcpu) irq_work_queue(&ecaps_report_irq_work); } +static void ecaps_offline_irq_workfn(struct irq_work *irq_work) +{ + wake_up(&ecaps_offline_waitq); +} + +static DEFINE_IRQ_WORK(ecaps_offline_irq_work, ecaps_offline_irq_workfn); + +/* + * @pcpu's sync marked by scx_offline_ecaps() has been processed or discarded. + * The waiter can't be woken under the rq lock, so the last completion wakes it + * through irq_work. The marks are all done while the cpu's hotplug thread waits + * in scx_offline_ecaps(), before hotplug parks any irq_work thread, so a + * regular irq_work is enough. + */ +static void ecaps_offline_done(struct rq *rq, struct scx_sched_pcpu *pcpu) +{ + lockdep_assert_rq_held(rq); + + pcpu->flags &= ~SCX_SCHED_PCPU_ECAPS_OFFLINE; + WRITE_ONCE(rq->scx.nr_ecaps_offline, rq->scx.nr_ecaps_offline - 1); + if (!rq->scx.nr_ecaps_offline) + irq_work_queue(&ecaps_offline_irq_work); +} + /** * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch * * pshard->caps[] is the target configuration. pcpu->ecaps is the effective - * transposed copy owned by the cid's cpu and written under @rq's lock. + * transposed copy owned by the cid's cpu and written only here under @rq's + * lock. + * + * A sched's ecaps change only with a delivery, so that whenever an op of the + * sched runs under @rq's lock, its ecaps are what its deliveries reported. A + * sync that can't deliver leaves them alone, and a later one that can reports + * the net change. * * A sched that newly gains baseline access here is owed an update_idle() so it * learns the cid's idle state. Such a gain arms the per-rq @@ -1131,20 +1158,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) if (likely(llist_empty(&rq->scx.ecaps_to_sync))) return; - /* - * ecaps are zeroed while the cpu is inactive and must stay zero. - * Discard queued syncs instead of processing them - the - * scx_online_ecaps() reseed re-syncs every sched on activation. - * cpu_active() clears before the offline zeroing and sets before the - * reseed is queued, so this test can neither miss a racing sync nor - * eat the reseed. - */ - if (unlikely(!cpu_active(cpu))) { - discard_queued_syncs(rq); - return; - } - - /* @cid is valid here: the cpu is active with queued syncs */ + /* @cid stays valid while a root is enabled */ cid = __scx_cpu_to_cid(cpu); batch = llist_del_all(&rq->scx.ecaps_to_sync); @@ -1153,6 +1167,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); struct scx_pshard *ps = cid_pshard(pcpu->sch, cid); bool report = pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT; + bool offline = pcpu->flags & SCX_SCHED_PCPU_ECAPS_OFFLINE; u64 old, ecaps, lost, gained; init_llist_node(pos); @@ -1167,6 +1182,10 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * set. Deliveries also need the cpu online to the root, see * handle_hotplug(). * + * An inactive cpu that can deliver is going down and reads as + * capless. A delivery below can drop the rq lock and the cpu's + * active state can change between syncs, so test it per sync. + * * A sync that can't deliver leaves the ecaps alone. It may * still carry a mark that is cleared below. Don't skip the rest * of the loop for it. @@ -1175,8 +1194,10 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) if (!(pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) || !(rq->scx.flags & SCX_RQ_OPS_ONLINE)) ecaps = old; - else + else if (cpu_active(cpu)) ecaps = calc_effective_caps(ps, cid); + else + ecaps = 0; WRITE_ONCE(pcpu->ecaps, ecaps); lost = old & ~ecaps; @@ -1193,13 +1214,8 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * scheduling can differ from @rq's cpu, so the context is set * up there. The rq recorded in it keeps the dispatches * targeting @rq. - * - * Both report the same before value, so one reported_ecaps - * covers them. */ - if (ecaps != pcpu->reported_ecaps && - (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) && - (rq->scx.flags & SCX_RQ_OPS_ONLINE)) { + if (ecaps != old) { struct scx_sched *parent = scx_parent(pcpu->sch); struct scx_dsp_ctx *dspc; @@ -1212,7 +1228,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; dspc->rq = rq; SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, - scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps); + scx_cpu_arg(cpu), old, ecaps); scx_flush_dispatch_buf(pcpu->sch, rq); } if (SCX_HAS_OP(parent, sub_child_ecaps_updated) && @@ -1221,15 +1237,16 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) dspc->rq = rq; SCX_CALL_OP(parent, sub_child_ecaps_updated, rq, pcpu->sch->ops.sub_cgroup_id, scx_cpu_arg(cpu), - pcpu->reported_ecaps, ecaps); + old, ecaps); scx_flush_dispatch_buf(parent, rq); } rq->scx.sub_dispatch_prev = NULL; - pcpu->reported_ecaps = ecaps; } if (report) ecaps_report_done(pcpu); + if (offline) + ecaps_offline_done(rq, pcpu); /* * Gaining baseline access owes an update_idle() so the sched @@ -1275,11 +1292,28 @@ void scx_init_rq_ops_online(void) } } +/* + * Make @rq's cpu dispatch through the ext class to process its ecaps syncs. + * queue_sync_ecaps() reschedules only when the cpu has no sync queued yet, + * never an inactive cpu, and not while the root bypasses, so reschedule here. + * The callers run on the cpu's hotplug thread or in ecaps_report(), never in + * the PM syscore phase. A higher class task can hold the cpu, so also start the + * ext server, which picks the ext class within its period. + */ +static void resched_sync_ecaps(struct rq *rq) +{ + lockdep_assert_rq_held(rq); + + update_rq_clock(rq); + dl_server_start(&rq->ext_server); + scx_resched_curr_deferred(rq); +} + /* * A cpu came back and the root's ops.cpu_online() has run. Let the cpu deliver * ecaps again and reseed each sub-sched's ecaps on its cid. The sync recomputes - * effective caps from the pshard and fires ops.sub_ecaps_updated() only on a - * real change since offline. + * effective caps from the pshard and reports any change since the last + * delivery. */ void scx_online_ecaps(struct rq *rq) { @@ -1313,40 +1347,69 @@ void scx_online_ecaps(struct rq *rq) guard(raw_spinlock)(&ps->lock); queue_sync_ecaps(pos, cid); } + + resched_sync_ecaps(rq); } /* - * A cpu is going down, and the root's ops.cpu_offline() follows. Stop the cpu's - * ecaps deliveries and zero each sub-sched's in-effect ecaps so cap checks - * treat the cpu as capless while offline. Pending and late-queued syncs are - * discarded at consumption by scx_process_sync_ecaps() while the cpu is - * inactive. Leave reported_ecaps. Ownership is unchanged, so the - * scx_online_ecaps() reseed reports only a genuine delta. No callback fires - * here. + * A cpu is going down, and the root's ops.cpu_offline() follows. Queue a marked + * sync for each sub-sched to report the loss and wait until each has been + * processed. Then stop the cpu's ecaps deliveries, so that none comes after + * ops.cpu_offline(). + * + * This runs on the cpu's hotplug thread. While it sleeps, the cpu keeps + * scheduling, and a pick through the ext class processes the syncs, see + * resched_sync_ecaps(). Only the marked syncs count, so a flood of changes on + * the cid can't hold up the hotplug. After the wait, every sub-sched's ecaps on + * the cpu are zero, and the later syncs have nothing to deliver. */ void scx_offline_ecaps(struct rq *rq) { s32 cpu = cpu_of(rq); struct scx_sched *root, *pos; + u32 nr = 0; + s32 cid; - guard(rq_lock_irqsave)(rq); + /* + * ecaps and the tables need a live root, see scx_online_ecaps(). The + * ext class isn't picked without one, so the syncs wouldn't be + * processed and the wait would never end. scx_enabled() can't change + * during hotplug. + */ + if (scx_enabled()) { + scoped_guard (rq_lock_irqsave, rq) { + root = scx_root_protected(); + cid = __scx_cpu_to_cid(cpu); - root = scx_root_protected(); + scx_for_each_descendant_pre(pos, root) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + struct scx_pshard *ps; - scx_for_each_descendant_pre(pos, root) { - struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + /* root holds every cap and never uses ecaps */ + if (!pos->level) + continue; - /* root holds every cap and never uses ecaps */ - if (!pos->level) - continue; + /* + * A batch that has yet to reach the sync sees + * the mark. A sync already being processed has + * left the list and is queued again below. + */ + pcpu->flags |= SCX_SCHED_PCPU_ECAPS_OFFLINE; + nr++; - WRITE_ONCE(pcpu->ecaps, 0); + ps = cid_pshard(pos, cid); + guard(raw_spinlock)(&ps->lock); + queue_sync_ecaps(pos, cid); + } - /* deliveries have stopped on this cpu, clear its mark */ - if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT) - ecaps_report_done(pcpu); + WRITE_ONCE(rq->scx.nr_ecaps_offline, nr); + resched_sync_ecaps(rq); + } + + wait_event(ecaps_offline_waitq, !READ_ONCE(rq->scx.nr_ecaps_offline)); } + guard(rq_lock_irqsave)(rq); rq->scx.flags &= ~SCX_RQ_OPS_ONLINE; } @@ -1357,7 +1420,7 @@ void scx_offline_ecaps(struct rq *rq) */ static void clear_all_caps(struct scx_sched *sch) { - s32 si, cpu; + s32 si; u32 cap_bit; lockdep_assert_cpus_held(); @@ -1373,16 +1436,6 @@ static void clear_all_caps(struct scx_sched *sch) for (cap_bit = 0; cap_bit < __SCX_NR_CAPS; cap_bit++) scx_cmask_clear(&ps->caps[cap_bit].cmask); } - - for_each_possible_cpu(cpu) { - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - - guard(rq_lock_irqsave)(cpu_rq(cpu)); - - /* drop an unreported loss, as the report skips inactive cpus */ - if (!cpu_active(cpu)) - pcpu->reported_ecaps = 0; - } } /* @@ -1413,6 +1466,9 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) { if (pos == &pcpu->ecaps_to_sync_node) { init_llist_node(pos); + /* a cpu going down may be waiting for the sync */ + if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_OFFLINE) + ecaps_offline_done(rq, pcpu); } else { pos->next = head; head = pos; @@ -1440,9 +1496,11 @@ void scx_discard_stale_ecaps_syncs(void) for_each_possible_cpu(cpu) { struct rq *rq = cpu_rq(cpu); + struct llist_node *pos, *tmp; guard(rq_lock_irqsave)(rq); - discard_queued_syncs(rq); + llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) + init_llist_node(pos); } } @@ -1748,8 +1806,14 @@ static void ecaps_enable(struct scx_sched *sch) * @sch: sub-scheduler to report * * Mark every cpu @sch was ever granted a cap on, deliver its pending change - * through the dispatch path and wait for it. An inactive cpu is skipped. A cpu - * going down meanwhile has its mark cleared by scx_offline_ecaps(). + * through the dispatch path and wait for it. An inactive cpu is skipped. It may + * already be offline, where the mark would never be processed and the wait + * would never end. On attach, the reseed at its activation reports its caps. On + * detach, its loss was reported as it went down. + * + * If a marked cpu goes down meanwhile, it clears the mark while still online, + * as ecaps_report_irq_work would never run on an offline cpu. A cpu going down + * waits for the marked sync in scx_offline_ecaps(). * * Called with scx_enable_mutex held. A sched's attach and detach reports must * not overlap, as the mark is one bit per pcpu. @@ -1768,7 +1832,6 @@ static void ecaps_report(struct scx_sched *sch) guard(rq_lock_irqsave)(rq); - /* syncs on an inactive cpu are discarded */ if (!cpu_active(cpu)) continue; @@ -1782,15 +1845,7 @@ static void ecaps_report(struct scx_sched *sch) pcpu->flags |= SCX_SCHED_PCPU_ECAPS_REPORT; atomic_inc(&sch->nr_ecaps_reports); queue_sync_ecaps(sch, cid); - - /* - * The report needs an ext dispatch on this cpu. A higher class - * task can hold the cpu, so also start the ext server, which - * picks the ext class within its period. - */ - update_rq_clock(rq); - dl_server_start(&rq->ext_server); - scx_resched_curr_deferred(rq); + resched_sync_ecaps(rq); } rcuwait_wait_event(&ecaps_report_wait, !atomic_read(&sch->nr_ecaps_reports), @@ -1822,9 +1877,9 @@ void scx_sub_disable(struct scx_sched *sch) /* * Clear @sch's caps and report the loss before any task goes back to * the parent. A disabling sched is granted nothing, so the caps stay - * cleared. Hold off cpu hotplug across both. The report covers every - * active cpu and the clear drops the unreported loss on the rest, so - * that no loss reaches the parent after ops.sub_detach(). + * cleared. Hold off cpu hotplug across both so that a cpu the report + * skips has finished going down. Its loss was reported as it went down, + * see scx_offline_ecaps(). * * An enable that failed before its attach report has nothing to report. * If it failed before scx_link_sched(), the root's disable doesn't wait diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 05cf0f0056e9..9e7993256c81 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -847,9 +847,10 @@ struct scx_rq { struct scx_sched *sched; struct llist_head ecaps_to_sync; /* pending ecaps syncs */ struct task_struct *sub_dispatch_prev; + u32 nr_ecaps_offline; /* marked syncs a down waits for */ #endif - cpumask_var_t cpus_to_sync; bool kick_sync_pending; + cpumask_var_t cpus_to_sync; unsigned long kick_sync; struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */ diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index e8e451be862d..63e3c9080e77 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -2180,35 +2180,16 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args) void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args) { u64 cgid = args->ops->sub_cgroup_id; - s32 nr_cids = qa.nr_cids; - s32 pos = qa.part.rr_pos; - u64 holder_cgid; - bool reset_pool; - s32 i, cid; - - if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || pos < 0 || pos >= MAX_PARTS) { - scx_bpf_error("-ERANGE"); - return; - } + s32 i; /* - * The child's caps are gone without a report, see ops.sub_detach(). - * cpuperf targets persist until the next write. Reset the child's excl - * cids, and the pool unless another child holds it: that child's revoke - * report or its own detach resets it then. + * qmap_sub_child_ecaps_updated() has reset the cpuperf targets on the + * loss reports. */ - holder_cgid = qa.part.rr_slots[pos]; - reset_pool = !holder_cgid || holder_cgid == cgid; - for (i = 0; i < MAX_SUB_SCHEDS; i++) { if (qa.sub_sched_ctxs[i].cgroup_id != cgid) continue; - bpf_arena_for(cid, 0, nr_cids) - if (cmask_test(cid, &qa.sub_sched_ctxs[i].granted_cids.mask) || - (reset_pool && cmask_test(cid, &qa.rr_cids.mask))) - scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE); - qa.sub_sched_ctxs[i].cgroup_id = 0; qa.sub_sched_ctxs[i].weight = 100; cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids); From ff343a94e44d0703ff011e9b68d461cb51959742 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 17 Oct 2026 15:59:52 -1000 Subject: [PATCH 06/34] sched_ext: Leave suspend and hibernation out of the ecaps reports Suspend and hibernation take down every non-boot cpu and bring them back. A loss and a regain per cpu to every sub-scheduler would be noise. Leave them out: a cpu they take down reports no loss and keeps the caps last delivered, and the reseed after resume reports only the net change. Their downs are the ones with cpuhp_tasks_frozen set between the PM notifications. A kexec jump sets the flag too but sends no PM notification. Two things need care. A cpu that suspend or hibernation takes down doesn't wait for its syncs, and no ecaps_report() mark can be pending on it then. The attach report runs while the attaching task waits uninterruptibly, which holds off the freezer, and the detach report holds cpus_read_lock(). The detach report reaches only active cpus, and the loss on a cpu that suspend or hibernation took down was never delivered. The detach waits for those cpus to come back, so every loss is delivered. They come back before the tasks thaw, so the wait doesn't hold up the freezer. If one fails to come back, the hierarchy is aborted at the end of the PM cycle. v2: Fix why no report mark is pending at a PM down, add a data_race(). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 15 ++++ kernel/sched/ext/internal.h | 5 +- kernel/sched/ext/sub.c | 146 +++++++++++++++++++++++++++++++++--- kernel/sched/ext/sub.h | 6 ++ 4 files changed, 159 insertions(+), 13 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 02276b5b374b..aa00b7537182 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -4170,6 +4170,7 @@ static void handle_hotplug(struct rq *rq, bool online) s32 cpu_or_cid = cpu; atomic_long_inc(&scx_hotplug_seq); + scx_update_frozen_cpus(cpu, online); /* * scx_root updates are protected by cpus_read_lock() and will stay @@ -9482,6 +9483,20 @@ static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void * { struct scx_sched *sch; + /* track the PM cycle whether or not a root is loaded */ + switch (event) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + case PM_RESTORE_PREPARE: + scx_sub_pm_begin(); + break; + case PM_POST_HIBERNATION: + case PM_POST_SUSPEND: + case PM_POST_RESTORE: + scx_sub_pm_end(); + break; + } + guard(rcu)(); sch = rcu_dereference(scx_root); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 4b9656848085..80c746054ce7 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -934,7 +934,10 @@ struct sched_ext_ops { * Deliveries for a cid happen only while the root considers the cid's * cpu online, never before the root's ops.cpu_online() or after its * ops.cpu_offline() for the cpu. A cpu going offline reports the loss - * of every cap on it, and a cpu coming back reports the regain. + * of every cap on it, and a cpu coming back reports the regain. A cpu + * that suspend or hibernation takes down reports no loss and keeps its + * caps while down. Only its net change, if any, is reported once it is + * back. */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index d88a436f8963..f7f33fd0c5e9 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -56,6 +56,19 @@ static struct rcuwait ecaps_report_wait = __RCUWAIT_INITIALIZER(ecaps_report_wai /* a cpu going down waits here for its marked syncs, see scx_offline_ecaps() */ static DECLARE_WAIT_QUEUE_HEAD(ecaps_offline_waitq); +/* + * The cpus that suspend or hibernation took down and that haven't come back. + * Their losses are not delivered, see scx_offline_ecaps(). Hotplug updates the + * set under cpus_write_lock(), and scx_sub_pm_end() clears it under + * cpus_read_lock(). A cpu that fails to come back stays in the set until + * regular hotplug brings it back or the PM cycle ends. + */ +static struct cpumask scx_frozen_cpus; +static DECLARE_WAIT_QUEUE_HEAD(scx_frozen_cpus_waitq); + +/* between the PREPARE and POST notifications of a PM cycle */ +static bool scx_pm_in_progress; + /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk * @pos: current position @@ -1130,6 +1143,17 @@ static void ecaps_offline_done(struct rq *rq, struct scx_sched_pcpu *pcpu) irq_work_queue(&ecaps_offline_irq_work); } +/* + * Test whether the hotplug operation in progress is suspend or hibernation + * taking a cpu down or bringing it back. A kexec jump also sets + * cpuhp_tasks_frozen but sends no PM notification, so outside a PM cycle its + * cpu downs deliver their losses as in regular hotplug. + */ +static bool scx_pm_frozen(void) +{ + return cpuhp_tasks_frozen && READ_ONCE(scx_pm_in_progress); +} + /** * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq @@ -1185,6 +1209,8 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * An inactive cpu that can deliver is going down and reads as * capless. A delivery below can drop the rq lock and the cpu's * active state can change between syncs, so test it per sync. + * Suspend and hibernation don't deliver the loss of a cpu they + * take down, see scx_offline_ecaps(). * * A sync that can't deliver leaves the ecaps alone. It may * still carry a mark that is cleared below. Don't skip the rest @@ -1196,6 +1222,8 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) ecaps = old; else if (cpu_active(cpu)) ecaps = calc_effective_caps(ps, cid); + else if (scx_pm_frozen()) + ecaps = old; else ecaps = 0; WRITE_ONCE(pcpu->ecaps, ecaps); @@ -1270,6 +1298,64 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE); } +void scx_sub_pm_begin(void) +{ + WRITE_ONCE(scx_pm_in_progress, true); +} + +/* + * Called on every cpu down and up whether or not a root is loaded, so that the + * set stays exact when a root is enabled or disabled during a PM cycle. A cpu + * that failed to come back from suspend or hibernation can come back through + * regular hotplug before the PM cycle ends, e.g. through the SMT control. The + * reseed at its activation reports the net change, so any up removes the cpu + * from the set. + */ +void scx_update_frozen_cpus(s32 cpu, bool online) +{ + if (online) { + if (!cpumask_test_cpu(cpu, &scx_frozen_cpus)) + return; + __cpumask_clear_cpu(cpu, &scx_frozen_cpus); + if (cpumask_empty(&scx_frozen_cpus)) + wake_up_all(&scx_frozen_cpus_waitq); + } else if (scx_pm_frozen()) { + __cpumask_set_cpu(cpu, &scx_frozen_cpus); + } +} + +/* + * The PM cycle is over. A cpu still in the set failed to come back, and its + * loss can't be delivered anymore. That breaks the guarantee that every ecaps + * change reaches the sub-scheds and their parents, so abort every sched. A root + * without sub-scheds is owed no reports and keeps running. + * + * Abort before clearing the set and waking the detaches in scx_sub_disable(), + * so that a parent whose ops.sub_detach() runs with that loss unreported is + * already being disabled, which ops.sub_detach() allows for. No exit cpu is + * recorded, as the cpu running this isn't the one that failed. + */ +void scx_sub_pm_end(void) +{ + /* hotplug updates the set under cpus_write_lock() */ + guard(cpus_read_lock)(); + + if (unlikely(!cpumask_empty(&scx_frozen_cpus)) && scx_has_subs()) { + struct scx_sched *root; + + guard(rcu)(); + root = rcu_dereference(scx_root); + if (root) + __scx_exit(root, SCX_EXIT_ERROR, 0, -1, + "%u cpu(s) failed to come back after suspend or hibernation", + cpumask_weight(&scx_frozen_cpus)); + } + + WRITE_ONCE(scx_pm_in_progress, false); + cpumask_clear(&scx_frozen_cpus); + wake_up_all(&scx_frozen_cpus_waitq); +} + /* * Called with cpus_read_lock() held before the root is published. The cpus * active now are online to the root without ops.cpu_online(), so they can @@ -1362,6 +1448,11 @@ void scx_online_ecaps(struct rq *rq) * resched_sync_ecaps(). Only the marked syncs count, so a flood of changes on * the cid can't hold up the hotplug. After the wait, every sub-sched's ecaps on * the cpu are zero, and the later syncs have nothing to deliver. + * + * The loss on a cpu that suspend or hibernation takes down is not delivered, so + * that the scheds don't see a PM cycle as cpus leaving and coming back. The cpu + * keeps the caps last delivered until it comes back, and the reseed then + * reports the net change. */ void scx_offline_ecaps(struct rq *rq) { @@ -1376,7 +1467,7 @@ void scx_offline_ecaps(struct rq *rq) * processed and the wait would never end. scx_enabled() can't change * during hotplug. */ - if (scx_enabled()) { + if (scx_enabled() && !scx_pm_frozen()) { scoped_guard (rq_lock_irqsave, rq) { root = scx_root_protected(); cid = __scx_cpu_to_cid(cpu); @@ -1809,11 +1900,16 @@ static void ecaps_enable(struct scx_sched *sch) * through the dispatch path and wait for it. An inactive cpu is skipped. It may * already be offline, where the mark would never be processed and the wait * would never end. On attach, the reseed at its activation reports its caps. On - * detach, its loss was reported as it went down. + * detach, its loss was reported as it went down. A cpu that suspend or + * hibernation took down is back by then, or the hierarchy is aborting, see + * scx_sub_disable(). * - * If a marked cpu goes down meanwhile, it clears the mark while still online, - * as ecaps_report_irq_work would never run on an offline cpu. A cpu going down - * waits for the marked sync in scx_offline_ecaps(). + * If a marked cpu goes down meanwhile, a regular down waits for the marked sync + * in scx_offline_ecaps() while the cpu is still online, as + * ecaps_report_irq_work would never run on an offline cpu. Suspend and + * hibernation don't wait, but no report runs across their downs. The attach + * report runs while the attaching task waits uninterruptibly, which holds off + * the freezer, and the detach report holds cpus_read_lock(). * * Called with scx_enable_mutex held. A sched's attach and detach reports must * not overlap, as the mark is one bit per pcpu. @@ -1881,15 +1977,41 @@ void scx_sub_disable(struct scx_sched *sch) * skips has finished going down. Its loss was reported as it went down, * see scx_offline_ecaps(). * - * An enable that failed before its attach report has nothing to report. - * If it failed before scx_link_sched(), the root's disable doesn't wait - * for it and may have retired the cid tables that the report reads. + * The loss on a cpu that suspend or hibernation took down is not + * delivered. Wait for such cpus to come back so that the report reaches + * them. They go down only after the freezer has frozen the tasks and + * come back before the tasks thaw, so the wait doesn't hold up the + * freezer. If one fails to come back, the wait can last until the PM + * cycle ends, and scx_sub_pm_end() aborts every sched first. + * + * The check and the report run in one cpus_read_lock() section so that + * no cpu goes down for PM between them. scx_sub_pm_end() can still + * clear the set meanwhile, which is fine, as it aborts every sched + * first. The report also needs scx_enable_mutex, see ecaps_report(). A + * cpu coming back takes cpus_write_lock(), so the wait drops + * cpus_read_lock(). It drops scx_enable_mutex too so that other enables + * and disables don't wait behind the PM cycle, and retakes both in + * order. + * + * An enable that failed before its attach report has nothing to report + * and doesn't wait. If it failed before scx_link_sched(), the root's + * disable doesn't wait for it and may have retired the cid tables that + * the report reads. */ - scoped_guard(cpus_read_lock) { - clear_all_caps(sch); - if (sch->ecaps_enabled) - ecaps_report(sch); + cpus_read_lock(); + while (sch->ecaps_enabled && !data_race(cpumask_empty(&scx_frozen_cpus))) { + cpus_read_unlock(); + mutex_unlock(&scx_enable_mutex); + /* lockless hint, the loop retests */ + wait_event_idle(scx_frozen_cpus_waitq, + data_race(cpumask_empty(&scx_frozen_cpus))); + mutex_lock(&scx_enable_mutex); + cpus_read_lock(); } + clear_all_caps(sch); + if (sch->ecaps_enabled) + ecaps_report(sch); + cpus_read_unlock(); percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index 46c7e2a578d1..eb1dba6d51e2 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -32,6 +32,9 @@ void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); +void scx_sub_pm_begin(void); +void scx_update_frozen_cpus(s32 cpu, bool online); +void scx_sub_pm_end(void); void scx_init_rq_ops_online(void); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); @@ -110,6 +113,9 @@ static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } static inline void scx_init_root_caps(struct scx_sched *sch) {} static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} +static inline void scx_sub_pm_begin(void) {} +static inline void scx_update_frozen_cpus(s32 cpu, bool online) {} +static inline void scx_sub_pm_end(void) {} static inline void scx_init_rq_ops_online(void) {} static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} From 993e8fbfecbbf284b9644c6f10741b28fb914094 Mon Sep 17 00:00:00 2001 From: Michal Blaszczyk Date: Mon, 19 Oct 2026 12:41:45 +0000 Subject: [PATCH 07/34] sched_ext: Reset runnable_at when a task leaves its scheduler The watchdog measures how long a task has been waiting from p->scx.runnable_at, which set_task_runnable() resets when the task becomes runnable after it last ran. ops_dequeue() never asks for a reset, as the dequeue of a waiting task is temporary, e.g. for a migration or an attribute change. Leaving the scheduler isn't temporary. A task which is switched out of SCHED_EXT while it's waiting keeps its runnable_at. When it's switched back and becomes runnable, its wait appears to have started before it left, and the time it spent in another class counts towards the timeout. If the watchdog checks before the task runs, it aborts the scheduler: sched_ext: class_switch_stall: runner[418] failed to run for 3.010s Here, the task waited for a few milliseconds before it was switched to SCHED_OTHER, slept outside of SCHED_EXT for three seconds, and was charged for all of it as soon as it woke up on SCHED_EXT again. The same happens when scx_sub_enable_workfn() hands a waiting task over to a new sub-scheduler, which can then be blamed for time the task waited under the parent. scx_rehome_task() isn't affected, as it moves the task through SCX_TASK_INIT, which resets runnable_at. The stale runnable_at also skews the default core-sched ordering in scx_prio_less(). Reset runnable_at in scx_disable_task(), when the task leaves the control of its scheduler. Fixes: 8a010b81b3a5 ("sched_ext: Implement runnable task stall watchdog") Cc: stable@vger.kernel.org # v6.12+ Assisted-by: LLM Signed-off-by: Michal Blaszczyk Reviewed-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index aa00b7537182..6fec09fd0970 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2454,7 +2454,11 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) struct scx_sched *sch = scx_task_sched(p); unsigned long opss; - /* dequeue is always temporary, don't reset runnable_at */ + /* + * Don't reset runnable_at. If @p is waiting, the wait continues across + * temporary dequeues, e.g. for a migration or an attribute change. + * scx_disable_task() ends it if @p leaves its scheduler. + */ clr_task_runnable(p, false); retry: @@ -4554,6 +4558,14 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) scx_set_task_slice(p, 0); p->scx.reenq_cnt = 0; + /* + * If @p is runnable but not running, its wait started at runnable_at, + * which ops_dequeue() didn't reset. Leaving @sch ends the wait. Reset + * runnable_at the next time @p is enqueued on SCX, so that the watchdog + * doesn't count the time @p spends elsewhere. + */ + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; + /* * Verify the task is not in BPF scheduler's custody. If flag * transitions are consistent, the flag should always be clear From e2859566ffbc3a9f33308255114ccf167124e048 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Sun, 18 Oct 2026 18:24:15 +0200 Subject: [PATCH 08/34] sched_ext: Queue the kick_sync wait from set_next_task_scx() The SCX_KICK_WAIT wait is deferred to rq->scx.kick_sync_bal_cb, which a final pick queues on the local rq. Under core scheduling, a pick that returns a task isn't final: pick_next_task() goes on to pick the SMT siblings, and their dispatch can release the shared rq lock with the callback still queued. A CPU acquiring the rq lock in that window then finds a pending balance callback and triggers the warning in rq_pin_lock(). Fix this by queuing the callback from set_next_task_scx() on SNT_PICK and SNT_REPICK. put_prev_set_next_task() calls it once the pick is final, after the core-wide selection, so the rq lock can't be dropped with the callback queued. If the CPU ends up running idle or a higher class task, no SCX task runs there and there's nothing to wait for: kick_sync_pending stays set and the next pick of an SCX task waits before running it. Also, proxy exec zaps the balance callbacks and picks again when find_proxy_task() fails, so clear kick_sync_pending when the callback runs rather than when it's queued. Fixes: 415cb193bb97 ("sched_ext: Fix SCX_KICK_WAIT deadlock by deferring wait to balance callback") Cc: stable@vger.kernel.org # v7.0+: c72945693b90: sched: Restart fair hrtick after same-task repicks Cc: stable@vger.kernel.org # v7.0+ Suggested-by: Tejun Heo Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 50 ++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 6fec09fd0970..5efc3ad1b119 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3422,6 +3422,8 @@ static void scx_start_task_running(struct rq *rq, struct task_struct *p) p->scx.flags |= SCX_TASK_RUN_TRACKED; } +static void kick_sync_wait_bal_cb(struct rq *rq); + static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e type) { bool first = type == SNT_PICK; @@ -3437,6 +3439,20 @@ static void set_next_task_scx(struct rq *rq, struct task_struct *p, enum snt_e t (type == SNT_PICK || type == SNT_REPICK)) rq->scx.flags |= SCX_RQ_PROXY_PENDING; + /* + * Defer the kick_sync wait to a balance callback which can drop rq lock + * and enable IRQs. Waiting directly in the pick path would deadlock + * against CPUs sending us IPIs (e.g., TLB flushes) while we wait for + * them. + * + * Queue it only once the pick is final, so that the rq lock can't be + * released with the callback queued (e.g., by another dispatch during + * a core-wide selection). If no SCX task gets picked, there's nothing + * to wait for and kick_sync_pending stays set for the next SCX pick. + */ + if (unlikely(rq->scx.kick_sync_pending) && (type == SNT_PICK || type == SNT_REPICK)) + queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, kick_sync_wait_bal_cb); + if (type == SNT_REPICK) return; @@ -3717,17 +3733,22 @@ static void kick_sync_wait_bal_cb(struct rq *rq) s32 cpu; /* - * This callback is queued and normally flushed within @rq's own - * scheduling pass. However, dispatch can drop the rq lock while it sits - * queued, and lock takers in that window (the sched class change paths, - * the scx task iterator) flush pending balance callbacks on release, - * running this one on a foreign CPU whose snapshots are unrelated. The - * kicked CPUs are already on their way to advance the kick_syncs being - * waited on. Don't get in the way. + * This callback is queued once the pick is final and flushed within + * @rq's own scheduling pass. If a lock taker on a foreign CPU ever + * flushes it, the snapshots are unrelated to that CPU. Don't get in the + * way and leave the wait pending for @rq's next pick. */ if (unlikely(cpu_of(rq) != smp_processor_id())) return; + /* + * Clear only when the wait actually runs. The callback can be dropped + * without running (e.g., proxy exec zaps the balance callbacks and + * picks again when find_proxy_task() fails), in which case the next + * pick must queue it again. + */ + rq->scx.kick_sync_pending = false; + ks = __this_cpu_read(scx_kick_syncs); ksyncs = rcu_dereference_sched(ks)->syncs; @@ -3926,25 +3947,12 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx) * NULL pick from the dl_server, the core continues picking and can * release the rq lock (e.g., prev_balance() or another dispatch). In * that case, run the deferred work from irq_work, which doesn't depend - * on the current rq lock section, and leave the kick sync wait pending - * for the next pick. + * on the current rq lock section. */ if (p != RETRY_TASK && (p || !force_scx)) { /* under core scheduling, see dispatch_core_pick() */ if (!core_sched) maybe_queue_balance_callback(rq); - - /* - * Defer to a balance callback which can drop rq lock and enable - * IRQs. Waiting directly in the pick path would deadlock against - * CPUs sending us IPIs (e.g., TLB flushes) while we wait for - * them. - */ - if (unlikely(rq->scx.kick_sync_pending)) { - rq->scx.kick_sync_pending = false; - queue_balance_callback(rq, &rq->scx.kick_sync_bal_cb, - kick_sync_wait_bal_cb); - } } else if (!core_sched && (rq->scx.flags & SCX_RQ_BAL_CB_PENDING)) { rq->scx.flags &= ~SCX_RQ_BAL_CB_PENDING; schedule_deferred(rq); From 9b5ce4345bf4d01dbf9ae5f6f6b1fb9cfdfc7b48 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Tue, 20 Oct 2026 10:43:20 +0200 Subject: [PATCH 09/34] sched_ext: Use irq_work for deferred actions with proxy execution maybe_queue_balance_callback() clears SCX_RQ_BAL_CB_PENDING when it queues the deferred callback. Proxy resolution can then zap the callback without running it, leaving the deferred actions unscheduled until a later wakeup or deferred request. Use the existing irq_work fallback for dispatch-time deferred requests when proxy execution is enabled. This keeps the work scheduled even if proxy resolution retries the pick or falls back to idle, without relying on another SCX dispatch to restore a discarded callback. Fixes: a8ad873113d3 ("sched_ext: defer queue_balance_callback() until after ops.dispatch") Reported-by: Tejun Heo Link: https://lore.kernel.org/r/08cac02c604dd73ef96fc92eb29d692b@kernel.org Signed-off-by: Andrea Righi Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 5efc3ad1b119..4b7d2602b4d4 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1129,9 +1129,10 @@ static void schedule_deferred_locked(struct rq *rq) return; /* - * If in dispatch, the balance callbacks will be called before rq lock - * is released. Schedule one. - * + * If in dispatch without proxy execution, the balance callbacks will + * be called before rq lock is released. Schedule one. With proxy + * execution, proxy resolution can zap the callbacks without another + * SCX dispatch to requeue them, so use irq_work instead. * * We can't directly insert the callback into the * rq's list: The call can drop its lock and make the pending balance @@ -1139,7 +1140,7 @@ static void schedule_deferred_locked(struct rq *rq) * * Just let dispatch_one() know that it must do it itself. */ - if (rq->scx.flags & SCX_RQ_IN_DISPATCH) { + if ((rq->scx.flags & SCX_RQ_IN_DISPATCH) && !sched_proxy_exec()) { rq->scx.flags |= SCX_RQ_BAL_CB_PENDING; return; } From 3b1c7c74a6da97f86bdd31c66e8c592aa4372e16 Mon Sep 17 00:00:00 2001 From: Andrea Righi Date: Wed, 21 Oct 2026 09:05:35 +0200 Subject: [PATCH 10/34] sched/core: Avoid requeueing a singleton balance callback With proxy execution, __schedule() can call set_next_task_scx() with SNT_REPICK and then SNT_PICK for the same donor. Both calls queue the kick_sync balance callback while kick_sync_pending remains set. When the callback is the only list entry, its next pointer is NULL, so the existing duplicate check misses it. Queueing it again links it to itself and causes the wait to run twice. Also check whether the callback is already the head of the list to reject the second insertion. This covers the kick_sync callback because nothing else is queued between SNT_REPICK and SNT_PICK. Requeueing the tail of a longer list remains possible and is outside the scope of this fix. Fixes: e2859566ffbc ("sched_ext: Queue the kick_sync wait from set_next_task_scx()") Reported-by: Tejun Heo Link: https://lore.kernel.org/r/08cac02c604dd73ef96fc92eb29d692b@kernel.org Signed-off-by: Andrea Righi Acked-by: Tejun Heo Acked-by: Peter Zijlstra (Intel) Signed-off-by: Tejun Heo --- kernel/sched/sched.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 9e7993256c81..628765910934 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2175,7 +2175,8 @@ queue_balance_callback(struct rq *rq, * balance_push() is active, see the comment with * balance_push_callback. */ - if (unlikely(head->next || rq->balance_callback == &balance_push_callback)) + if (unlikely(head->next || rq->balance_callback == head || + rq->balance_callback == &balance_push_callback)) return; head->func = func; From 651694bc47a15b99e756ebb27f4e6be6d59bf397 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Wed, 21 Oct 2026 18:12:14 +0900 Subject: [PATCH 11/34] sched_ext: Reject oversized automatic CID shards Automatic topology packing starts a new shard before adding a core when the current shard would exceed its target size. However, a first core is always accepted so that cores remain indivisible. An indivisible core with more than SCX_CID_SHARD_MAX_CPUS CPUs cannot be represented without exceeding the hard shard limit. Per-shard state uses fixed-size inline cmask storage sized for SCX_CID_SHARD_MAX_CPUS. Passing an oversized shard to alloc_pshard() can therefore make cmask initialization overwrite the following fields. Reject an indivisible core larger than SCX_CID_SHARD_MAX_CPUS before adding it to a shard. Existing non-empty shard splitting already keeps the total within the hard cap, while an indivisible core may still exceed cid_shard_size when it fits within the hard limit. Report the oversized core and the hard limit with scx_error() so that the rejection is diagnosable. tj: The bug is only reproducible with artificial VM topologies. It can't happen on real hardware. Fixes: e2c841b91bf9 ("sched_ext: Add CID sharding") Reviewed-by: Andrea Righi Tested-by: Andrea Righi Signed-off-by: Hui Su Signed-off-by: Tejun Heo --- kernel/sched/ext/cid.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/sched/ext/cid.c b/kernel/sched/ext/cid.c index 9fc219257004..20e40e3fe8ff 100644 --- a/kernel/sched/ext/cid.c +++ b/kernel/sched/ext/cid.c @@ -309,6 +309,12 @@ s32 scx_cid_init(struct scx_sched *sch) * empty shard. */ cids_in_core = cpumask_weight(core_scratch); + if (cids_in_core > SCX_CID_SHARD_MAX_CPUS) { + scx_error(sch, "core with %u cpus exceeds max shard size %d", + cids_in_core, SCX_CID_SHARD_MAX_CPUS); + return -E2BIG; + } + max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0); if (cores_in_shard && (cores_in_shard >= max_cores || From 93b1512ca69fbe3da66bbd2bab76e9ed74277679 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:08:59 -1000 Subject: [PATCH 12/34] sched_ext: Fix the stall watchdog testing and blaming the wrong sched under bypass While a sched is bypassing, its tasks are run by its host, the nearest ancestor that isn't bypassing. check_rq_for_timeouts() tests a runnable task against its owner's timeout. Since 2c91169377f4 ("sched_ext: Blame the DSQ's owning scheduler for a runnable stall"), it ejects the sched owning the DSQ the task waits on. The sched tested, the sched ejected and the sched running the task can all differ. - Timeouts are per sched. A bypassing sub-scheduler with a 2s timeout under a root with the default 30s has its tasks on the root's bypass DSQ. The root, not bypassing itself, consumes that DSQ before calling ops.dispatch() only on every other pass, and on the others only after ops.dispatch() yields nothing. A task that waits 2s there ejects the root, disabling all of sched_ext. - A bypassing owner's task on the local DSQ is blamed on the owner, which places nothing while bypassing. If the owner is already exiting, e.g. during its own disable, the exit is a no-op and nothing is ejected. If the host isn't running the task, the owner's disable, which waits on it, may never finish. - The wait isn't restarted when bypass moves a task. The time it waited under its owner counts against the host once it is on the host's bypass DSQ, and its time there against the owner once bypass lifts. The sched running a task is its owner, or the host while the owner is bypassing. Test the task against that sched's timeout and blame it. A bypass requeue can change the sched running a task, so restart the wait of every task scx_bypass() requeues except the root's own, which the root runs bypassing or not. Single-scheduler behavior is unchanged. A task that migrates while scx_bypass() works through the cpus, leaving one before its requeue and arriving on another after it, is requeued by neither and keeps its wait, which then counts against the sched now running it. That is what happened to every task before this patch. This might be worth addressing in the future, but the race is rare and the real-world implications should be limited. Fixes: 2c91169377f4 ("sched_ext: Blame the DSQ's owning scheduler for a runnable stall") Cc: stable@vger.kernel.org # v7.3+: 30fb2b1e4d41: sched_ext: Fix the running task keeping its protection into bypass Cc: stable@vger.kernel.org # v7.3+: 5809f749fbdb: sched_ext: Fix a bypassed task stalling on a cpu its scheduler has no caps on Cc: stable@vger.kernel.org # v7.3+ Reported-by: Andrea Righi Link: https://lore.kernel.org/all/at4l0yXTk7T_ouk0@gpd4/ Reported-by: Sashiko Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 4b7d2602b4d4..a9621bbdb183 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2458,7 +2458,8 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) /* * Don't reset runnable_at. If @p is waiting, the wait continues across * temporary dequeues, e.g. for a migration or an attribute change. - * scx_disable_task() ends it if @p leaves its scheduler. + * scx_disable_task() ends it if @p leaves its scheduler. scx_bypass() + * restarts it when its requeue can change the sched running @p. */ clr_task_runnable(p, false); @@ -4298,24 +4299,17 @@ static bool check_rq_for_timeouts(struct rq *rq) goto out_unlock; list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) { - struct scx_sched *sch = scx_task_sched(p); + /* + * @p's owner schedules @p, or its host while the owner is + * bypassing. Test @p against that sched's timeout and blame it. + */ + struct scx_sched *sch = scx_exec_sched(scx_task_sched(p), cpu_of(rq)); unsigned long last_runnable = p->scx.runnable_at; if (unlikely(time_after(jiffies, last_runnable + READ_ONCE(sch->watchdog_timeout)))) { - struct scx_dispatch_q *dsq = READ_ONCE(p->scx.dsq); u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable); - /* - * A task can be stuck on a DSQ that a sched other than - * its owner is responsible for draining, e.g. an - * ancestor's bypass DSQ while the owner is bypassing. - * Blame the drainer. The local DSQ is consumed by the - * cpu itself and keeps blame on the owner. - */ - if (dsq && dsq->sched && dsq->id != SCX_DSQ_LOCAL) - sch = dsq->sched; - __scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq), "%s[%d] failed to run for %u.%03us", p->comm, p->pid, dur_ms / 1000, @@ -6976,12 +6970,26 @@ void scx_bypass(struct scx_sched *sch, bool bypass) */ list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list, scx.runnable_node) { - if (!scx_is_descendant(scx_task_sched(p), sch)) + struct scx_sched *owner = scx_task_sched(p); + + if (!scx_is_descendant(owner, sch)) continue; /* cycling deq/enq is enough, see the function comment */ scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { - /* nothing */ ; + /* + * The requeue can change the sched running @p, + * and the watchdog counts @p's wait against + * that sched. Restart the wait unless @owner is + * the root, which runs its own tasks bypassing + * or not. + * + * A dispatch in flight writes p->scx.flags + * without @p's rq lock. The dequeue waits it + * out, so set the flag after it. + */ + if (scx_parent(owner)) + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; } } From b07a7450f6ef98c79ebd450a92fc4bb6b8f1e02e Mon Sep 17 00:00:00 2001 From: Junbin Zhang Date: Tue, 27 Oct 2026 13:16:34 +0800 Subject: [PATCH 13/34] sched_ext: Reject rq-lock-taking kfuncs in tracing programs scx_bpf_cpuperf_set() and scx_bpf_cidperf_set() are members of scx_kfunc_ids_any, which is registered for BPF_PROG_TYPE_TRACING. A tracing program can run in a context which already holds the target rq lock, in which case scx_cpuperf_set() attempts to acquire the same lock again: __schedule() rq_lock(rq); /* rq->__lock is held */ trace_sched_switch(); BPF tracing program scx_bpf_cpuperf_set(smp_processor_id(), 1024); scx_cpuperf_set(); locked_rq = scx_locked_rq(); /* NULL */ rq_lock_irqsave(rq, &rf); /* same lock */ scx_locked_rq() only tracks rq locks held during SCX callback invocations, so it cannot detect an rq lock held by the surrounding kernel code. The deadlock can be reproduced with two BPF programs. The first is a minimal SCX scheduler, which only needs to be registered and enabled: SEC(".struct_ops.link") struct sched_ext_ops minimal_ops = { .name = "minimal", }; The second is a tracing program attached to tp_btf/sched_switch, which fires from __schedule() with rq->__lock already held: SEC("tp_btf/sched_switch") int BPF_PROG(on_sched_switch, bool preempt, struct task_struct *prev, struct task_struct *next, unsigned int prev_state) { scx_bpf_cpuperf_set(bpf_get_smp_processor_id(), 1024); return 0; } With both programs attached, the next context switch on any CPU deadlocks that CPU. Lockdep reports: WARNING: possible recursive locking detected 7.3.0-rc7 #3 Not tainted swapper/3/0 is trying to acquire lock: (&rq->__lock){-.-.}-{2:2}, at: raw_spin_rq_lock_nested+0x1e/0x30 but task is already holding lock: (&rq->__lock){-.-.}-{2:2}, at: __schedule+0xeb/0x1430 Possible unsafe locking scenario: CPU0 ---- lock(&rq->__lock); lock(&rq->__lock); *** DEADLOCK *** Call Trace: print_deadlock_bug.cold+0xc0/0xd0 __lock_acquire+0x12fa/0x1b80 lock_acquire+0xd1/0x2f0 _raw_spin_lock_nested+0x32/0x40 raw_spin_rq_lock_nested+0x1e/0x30 scx_cpuperf_set+0x1e9/0x310 scx_bpf_cpuperf_set+0x9b/0x230 bpf_prog_..._on_sched_switch+0x3d/0x41 bpf_trace_run4+0x125/0x450 __traceiter_sched_switch+0x4b/0x70 __schedule+0x94a/0x1430 schedule_idle+0x26/0x50 cpu_startup_entry+0x2d/0x30 start_secondary+0xfc/0x100 The CPU remains locked up with interrupts disabled. scx_bpf_cidperf_set() is similarly affected, as it resolves the CID through scx_cid_to_cpu() before reaching the same rq-locking path. There is no reliable way for these kfuncs to determine whether the surrounding tracing context already holds an rq lock. Reject calls to both kfuncs from tracing programs at verification time, while preserving their availability to SCX struct_ops and syscall programs. Fixes: a11d6784d731 ("sched_ext: Fix missing rq lock in scx_bpf_cpuperf_set()") Signed-off-by: Junbin Zhang Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index a9621bbdb183..3c3fb8c18eab 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -11830,6 +11830,22 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = { .filter = scx_kfunc_context_filter, }; +/* + * Kfuncs which may take the rq lock on their unlocked path. + * + * A tracing program can execute in any kernel context, including ones which + * already hold the rq lock (e.g. sched_switch). scx_locked_rq() only tracks + * rq locks taken for SCX callbacks, so from such a context it returns NULL + * and these kfuncs would acquire the already held rq lock again and deadlock + * the CPU. They cannot be made safe for tracing programs, so keep them off + * the tracing hook. Members must also stay in scx_kfunc_ids_any so that + * struct_ops and syscall programs keep their access. + */ +BTF_KFUNCS_START(scx_kfunc_ids_no_tracing) +BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) +BTF_KFUNCS_END(scx_kfunc_ids_no_tracing) + /* * cpu-form kfuncs that are forbidden from cid-form schedulers * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must @@ -11938,6 +11954,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id); bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id); bool in_cid = btf_id_set8_contains(&scx_kfunc_ids_cid, kfunc_id); + bool in_no_tracing = btf_id_set8_contains(&scx_kfunc_ids_no_tracing, kfunc_id); u32 moff, flags; /* Not an SCX kfunc - allow. */ @@ -11950,7 +11967,7 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) return (in_unlocked || in_select_cpu || in_idle || in_any || in_cid) ? 0 : -EACCES; if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) - return (in_any || in_idle || in_cid) ? 0 : -EACCES; + return (!in_no_tracing && (in_any || in_idle || in_cid)) ? 0 : -EACCES; /* * add_subprog_and_kfunc() collects all kfunc calls, including dead code From 2e68cfad33faad70187df89b4c9f93ae439b7946 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:32 -1000 Subject: [PATCH 14/34] sched_ext: Pack scx_ent_flags' multi-bit fields at the top The following patches add a flag for a kernel-owned slice to scx_ent_flags. The two 3-bit fields, the task state at bits 8 to 10 and the reenqueue reason at bits 12 to 14, sit right above the single-bit flags, so the new flag would have to take bit 11 or 15, out of sequence with the rest. Pack the fields at the top of the word, below SCX_TASK_CURSOR, the state at bits 25 to 27 and the reason at bits 28 to 30, so that the single-bit flags continue from bit 8. The renumbering changes the values a BPF scheduler reads from p->scx.flags. Nothing in the kernel or the in-tree schedulers hardcodes the numbers. The enum ABI table in tools/sched_ext/include/scx/enums_abi.autogen.h, kept for repairing enum values that a kernel's BTF truncates, is regenerated for the two fields. In the scx repository, scx_eevdf tests SCX_TASK_REENQ_IMMED and SCX_TASK_REENQ_PREEMPTED against SCX_TASK_REENQ_REASON_MASK. Those three names belong in its load-time enum table, so that they follow the running kernel. No behavior change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 11 +++++--- .../sched_ext/include/scx/enums_abi.autogen.h | 28 +++++++++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index efd635a99309..b341c328948e 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -107,7 +107,10 @@ enum scx_ent_flags { SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */ /* - * Bits 8 to 10 are used to carry task state: + * The multi-bit fields are packed at the top, below SCX_TASK_CURSOR, so + * that the single-bit flags stay contiguous from bit 0. + * + * Bits 25 to 27 are used to carry task state: * * NONE ops.init_task() not called yet * INIT_BEGIN ops.init_task() in flight; see sched_ext_dead() @@ -116,7 +119,7 @@ enum scx_ent_flags { * ENABLED fully initialized and in sched_ext * DEAD terminal state set by sched_ext_dead() */ - SCX_TASK_STATE_SHIFT = 8, + SCX_TASK_STATE_SHIFT = 25, SCX_TASK_STATE_BITS = 3, SCX_TASK_STATE_MASK = ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT, @@ -128,7 +131,7 @@ enum scx_ent_flags { SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT, /* - * Bits 12 to 14 are used to carry reenqueue reason. In addition to + * Bits 28 to 30 are used to carry reenqueue reason. In addition to * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues. * @@ -139,7 +142,7 @@ enum scx_ent_flags { * CAP sub-sched cap miss, see p->scx.reenq_reason_* * PROXY proxy state prevented a remote DSQ transfer */ - SCX_TASK_REENQ_REASON_SHIFT = 12, + SCX_TASK_REENQ_REASON_SHIFT = 28, SCX_TASK_REENQ_REASON_BITS = 3, SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT, diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index 1186daa9cb14..05a460611c4d 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -120,24 +120,24 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, { "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU }, - { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x19LLU }, { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, - { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0xe000000LLU }, { "scx_ent_flags", "SCX_TASK_NONE", 0x0LLU }, - { "scx_ent_flags", "SCX_TASK_INIT_BEGIN", 0x100LLU }, - { "scx_ent_flags", "SCX_TASK_INIT", 0x200LLU }, - { "scx_ent_flags", "SCX_TASK_READY", 0x300LLU }, - { "scx_ent_flags", "SCX_TASK_ENABLED", 0x400LLU }, - { "scx_ent_flags", "SCX_TASK_DEAD", 0x500LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0xcLLU }, + { "scx_ent_flags", "SCX_TASK_INIT_BEGIN", 0x2000000LLU }, + { "scx_ent_flags", "SCX_TASK_INIT", 0x4000000LLU }, + { "scx_ent_flags", "SCX_TASK_READY", 0x6000000LLU }, + { "scx_ent_flags", "SCX_TASK_ENABLED", 0x8000000LLU }, + { "scx_ent_flags", "SCX_TASK_DEAD", 0xa000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0x1cLLU }, { "scx_ent_flags", "SCX_TASK_REENQ_REASON_BITS", 0x3LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x7000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x70000000LLU }, { "scx_ent_flags", "SCX_TASK_REENQ_NONE", 0x0LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x1000LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU }, - { "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x5000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x10000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x20000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x30000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x40000000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x50000000LLU }, { "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU }, { "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU }, From 15c5a52c1f193159f0a2faa1e3b03d112125cc17 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:33 -1000 Subject: [PATCH 15/34] sched_ext: Clear holding_cpu where the claim is confirmed The bypass load balancer will take tasks off a remote cpu's bypass DSQ with unlink_dsq_and_switch_rq_lock() in the following patches and insert them into another bypass DSQ. Clearing holding_cpu after the claim is up to each claimant, so the balancer would need a clear of its own. unlink_dsq_and_switch_rq_lock() marks a task with holding_cpu for the gap between dropping the DSQ lock and taking the task's rq lock, so that a dequeue in that gap can tell the claimant it lost. Once the claimant holds the rq lock and finds the marker intact, nothing can take the task away under it and the marker has no reader left. Each claimant clears it at its own site after that point: - dispatch_to_local_dsq(), which runs the handshake inline, in its same-rq and fallback branches - scx_proxy_reject_task() for the proxy reject - the dequeue inside deactivate_task() for both remote moves Clear it where the claim is confirmed instead and drop the explicit clears. No behavior change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 19 +++++++++++-------- kernel/sched/ext/internal.h | 11 ++++++----- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3c3fb8c18eab..afd961aad11d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2865,7 +2865,6 @@ static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, lockdep_assert_rq_held(rq); WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); - p->scx.holding_cpu = -1; p->scx.flags |= SCX_TASK_REENQ_PROXY; scx_divert_strip_flags(p, &enq_flags); @@ -2899,8 +2898,9 @@ static void scx_proxy_reject_task(struct scx_sched *sch, struct rq *rq, * tell whether we lost to dequeue by testing whether the holding_cpu still * points to this CPU. See scx_dispatch_dequeue() for the counterpart. * - * On return, @dsq is unlocked and @src_rq is locked. Returns %true if @p is - * still valid and detached from @dsq. %false if lost to dequeue. + * On return, @dsq is unlocked and @src_rq is locked. Return %true if @p is + * still valid and detached from @dsq, with holding_cpu cleared. Return %false + * if @p was lost to a dequeue. */ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, struct scx_dispatch_q *dsq, @@ -2923,11 +2923,13 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, if (likely(p->scx.holding_cpu == cpu) && !WARN_ON_ONCE(src_rq != task_rq(p))) { /* * Keep ->dsq set until we own @src_rq so that a racing dequeue - * can find @dsq and clear holding_cpu. Once ownership is - * confirmed, clear it under @src_rq so that deactivate_task() - * takes the !dsq path instead of retaking @dsq->lock. + * can find @dsq and clear holding_cpu. Once we own @src_rq, the + * handshake is over. Clear ->dsq so that deactivate_task() + * takes the !dsq path instead of retaking @dsq->lock. Clear + * holding_cpu with it so that no caller has to. */ p->scx.dsq = NULL; + p->scx.holding_cpu = -1; scx_wait_dispatching(p); return true; } @@ -3163,20 +3165,21 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, !WARN_ON_ONCE(src_rq != task_rq(p))) { bool fallback = false; + /* the handshake is over, see unlink_dsq_and_switch_rq_lock() */ + p->scx.holding_cpu = -1; + /* * If @p is staying on the same rq, there's no need to go * through the full deactivate/activate cycle. Optimize by * abbreviating move_remote_task_to_local_dsq(). */ if (src_rq == dst_rq) { - p->scx.holding_cpu = -1; scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p, slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE); } else if (unlikely(task_proxy_unsafe_to_move(p))) { fallback = true; scx_proxy_reject_task(sch, src_rq, p, enq_flags); } else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) { - p->scx.holding_cpu = -1; fallback = true; scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), p, slice, vtime, diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 80c746054ce7..752a9a2a94b0 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2157,12 +2157,13 @@ static const char *scx_enable_state_str[] = { * 4. Lock src_rq (where the task currently lives) * 5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the * race (dequeue clears holding_cpu to -1 when it takes the task), in - * this case migration is aborted - * 6. If src_rq == dst_rq: clear holding_cpu and enqueue directly - * into dst_rq's local DSQ (no lock swap needed) + * this case migration is aborted. Otherwise the claim is confirmed and + * holding_cpu is cleared here, see unlink_dsq_and_switch_rq_lock() + * 6. If src_rq == dst_rq: enqueue directly into dst_rq's local DSQ (no lock + * swap needed) * 7. Otherwise, verify under src_rq lock that the task can be moved to dst_rq - * (CPU affinity, migration_disabled, etc.). If not, clear holding_cpu, - * leave the task on src_rq, and enqueue it on the fallback DSQ. + * (CPU affinity, migration_disabled, etc.). If not, leave the task on + * src_rq, and enqueue it on the fallback DSQ. * 8. Otherwise (i.e. if the task can be moved to dst_rq), call * move_remote_task_to_local_dsq(), which releases src_rq, locks dst_rq, * and performs the deactivate/activate migration cycle From 40c9ad7de866065ba823bcfc0c250a377b107106 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:34 -1000 Subject: [PATCH 16/34] sched_ext: Fix bypassed tasks stranded by the reenqueue limit scx_do_enqueue_task() counts a task's reenqueues until the task runs. Past SCX_REENQ_MAX_REPEAT it ejects the owning scheduler and leaves the task off every DSQ for the scheduler's exit to requeue. The count comes before the bypass test, so it also counts the reenqueues of a task whose scheduler is bypassing. Those reenqueues are not the scheduler's placements, and ejecting the scheduler for them fixes nothing. Under bypass the kernel queues the scheduler's tasks on the bypass DSQs. A host calling scx_bpf_reenqueue_local() reenqueues a bypassed child's tasks on the cpu's local DSQ along with its own, and the reject drain reenqueues a consumed task that a cap check rejected. In the following patches, the consume itself reenqueues a task it can't run right away, and the bypass load balancer reenqueues a task at its home. The count is reset only when the task runs or leaves sched_ext, so an ejection can lead to a stranding the exit does not recover: 1. the task's reenqueues pass the limit, which ejects its scheduler and strands the task 2. the exit's scx_bypass() cycles the task through dequeue and enqueue, which requeues it on a bypass DSQ with the count still past the limit 3. the task's next reenqueue under bypass strands it again 4. scx_bypass() requeues stranded tasks only when the scheduler wasn't bypassing yet, so the task stays off every DSQ until the disable walks the tasks 5. the disable takes scx_enable_mutex, the cgroup lock and scx_fork_rwsem before the walk, and a sub-scheduler's disable the cpu hotplug lock too. It hangs if the stranded task holds one of them, forking, attaching to a cgroup or doing a cpu hotplug Test bypass before the count. A task reenqueued under bypass stays queued until a later dispatch or balancer pass, so there is no tight loop to bound, and the task's first run resets the count. The offline test moves up with it to stay ahead of the bypass test. A reenqueue on an offline cpu goes to the local DSQ and isn't counted either. A host that keeps calling scx_bpf_reenqueue_local() could get a bypassed child ejected, as the child's tasks on the local DSQ counted against the child. The reenqueues are the host's, and its own tasks still count against it. v2: The description lists the cpu hotplug lock among the locks a sub-scheduler's disable takes before the walk. Fixes: 7706d6e4f2e3 ("sched_ext: Bound per-task reenqueues and eject the owning scheduler") Reported-by: Sashiko Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index afd961aad11d..38946df9aa61 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2216,11 +2216,31 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, */ p->scx.flags &= ~SCX_TASK_IMMED; + /* + * If !scx_rq_online(), we already told the BPF scheduler that the CPU + * is offline and are just running the hotplug path. Don't bother the + * BPF scheduler. + */ + if (!scx_rq_online(rq)) + goto local; + + if (scx_bypassing(sch, cpu_of(rq))) { + __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); + goto bypass; + } + /* * A task reenqueued too many times without running means the scheduler * keeps re-deciding a placement it can't honor, e.g. re-inserting to a * cid it lacks caps on. Eject the owning scheduler and strand the task * to be picked up during sched exit. + * + * Bypass guarantees that every runnable task keeps running, and the + * disable relies on that to take sleeping locks. The kernel places the + * tasks of a bypassing scheduler and those on an offline rq, and + * counting their reenqueues could strand a task under bypass and + * deadlock the disable on a lock the task holds, so both tests stay + * above the count. */ if (enq_flags & SCX_ENQ_REENQ) { if (++p->scx.reenq_cnt > 1) @@ -2234,19 +2254,6 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, } } - /* - * If !scx_rq_online(), we already told the BPF scheduler that the CPU - * is offline and are just running the hotplug path. Don't bother the - * BPF scheduler. - */ - if (!scx_rq_online(rq)) - goto local; - - if (scx_bypassing(sch, cpu_of(rq))) { - __scx_add_event(sch, SCX_EV_BYPASS_DISPATCH, 1); - goto bypass; - } - if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID) goto direct; @@ -4993,7 +5000,8 @@ static void process_ddsp_deferred_locals(struct rq *rq) * %SCX_TASK_REENQ_IMMED. If the BPF scheduler dispatches back to the same local * DSQ with %SCX_ENQ_IMMED while the CPU is still unavailable, this triggers * another reenq cycle. Repetitions are bounded by %SCX_REENQ_MAX_REPEAT in - * scx_do_enqueue_task(), which ejects the task's owning scheduler. + * scx_do_enqueue_task(), which ejects the task's owning scheduler. A bypassing + * scheduler's tasks are placed by the kernel instead and are not counted. */ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, u64 *reenq_flags, u32 *reason) @@ -5288,7 +5296,8 @@ static void process_deferred_reenq_users(struct rq *rq) * * A task can be re-rejected repeatedly. Reenqueues are bounded per task by * SCX_REENQ_MAX_REPEAT in scx_do_enqueue_task(), which ejects the owning - * scheduler. + * scheduler. A bypassing scheduler's tasks are placed by the kernel instead and + * are not counted. */ static void scx_reenq_reject(struct rq *rq) { From a676ada0d6544928c0ceb032fb991a9abe0977db Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:35 -1000 Subject: [PATCH 17/34] sched_ext: Fix bypassed tasks looping on a host without SCX_CAP_ENQ Bypass exists so that runnable tasks make progress without trusting the BPF scheduler. A host with SCX_CAP_ENQ_IMMED (BASE) but not SCX_CAP_ENQ on a cpu cannot run a single bypassed task there. The task loops instead: 1. a bypassed task lands on the host's bypass DSQ for the cpu 2. the cpu's dispatch descends into the host, which holds BASE there 3. the host consumes the task into the local DSQ as a plain insert 4. the insert needs ENQ and the host lacks it, so the task is rejected 5. the reject drain reenqueues the task and it is back at step 1 A bypass consume puts a task on the cpu being dispatched for, to run right away. That is the IMMED contract, and BASE is the cap it needs. Consume with SCX_ENQ_IMMED. BASE is then the one cap a bypassed task ever needs. Dispatch descends into a host only on a cpu where the host holds BASE, so a consume never lacks it. A consumed task is first on the local DSQ when dispatch enters the host with the DSQ empty. That is always the case for the root and normally for a parent dispatching into a child. A consumed task that cannot run right away is reenqueued like any IMMED task and goes back to the bypass DSQ. Waiting on the local DSQ would need ENQ. The task cannot run right away when a higher class or a HEAD insert took the cpu meanwhile, or when the parent queued its own task before dispatching into the host. Each such task waits on the bypass DSQ for the next dispatch. The reenqueue is not counted against the reenqueue limit, as the task's scheduler is bypassing. v2: The description says that the reenqueue of a consumed task is not counted against the reenqueue limit (sashiko). Fixes: 701b7bcad868 ("sched_ext: Add the SCX_CAP_ENQ cap") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/inlines.h | 15 ++++++++++++--- kernel/sched/ext/sub.c | 6 ++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/kernel/sched/ext/inlines.h b/kernel/sched/ext/inlines.h index 8de9d8fb4b25..1e39d12678c4 100644 --- a/kernel/sched/ext/inlines.h +++ b/kernel/sched/ext/inlines.h @@ -39,10 +39,18 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, if (scx_consume_global_dsq(sch, rq)) return SCX_DSP_LOCAL; + /* + * A bypass consume puts a task on the cpu being dispatched for, to run + * right away. That is an IMMED insert and needs only + * %SCX_CAP_ENQ_IMMED, the base cap. Dispatch descends into @sch only on + * a cpu where @sch holds the base cap, so @sch holds it here. The same + * holds for the consume at the end of this function. + */ if (scx_bypass_dsp_enabled(sch)) { /* if @sch is bypassing, only the bypass DSQs are active */ if (scx_bypassing(sch, cpu)) { - if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) + if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), + SCX_ENQ_IMMED)) return SCX_DSP_LOCAL; return SCX_DSP_NONE; } @@ -63,7 +71,8 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), + SCX_ENQ_IMMED)) { __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; } @@ -135,7 +144,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq, * scheduler's ops.dispatch() doesn't yield any tasks. */ if (scx_bypass_dsp_enabled(sch) && - scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) { + scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), SCX_ENQ_IMMED)) { __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1); return SCX_DSP_LOCAL; } diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index f7f33fd0c5e9..a7b14ea16822 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -2722,6 +2722,12 @@ __bpf_kfunc_start_defs(); * child schedulers. The child scheduler runs its dispatch operation to * move tasks from dispatch queues to the local runqueue. * + * Dispatch into a child while the local DSQ is empty, when the cpu needs a + * task. A child holding only %SCX_CAP_ENQ_IMMED can insert only IMMED tasks, + * and the consume of its bypassing descendants' bypass DSQs is one such insert. + * With a task already queued on the local DSQ, such an insert fails and the + * task is reenqueued. + * * Returns: true on success, false if cgroup_id is invalid, not a direct * child, or caller lacks dispatch permission. */ From 867d0ac6d730768610eae7fc41f5dd3b70b8d1a4 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:36 -1000 Subject: [PATCH 18/34] sched_ext: Move refill_task_slice_dfl() below apply_slice_vtime() Move refill_task_slice_dfl() below apply_slice_vtime(), with the other slice writers, to prepare for the kernel-slice handling the following patches add to them. No functional change. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 38946df9aa61..d1404a4e1564 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1569,6 +1569,16 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e p->scx.dsq_vtime = vtime; } +static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) +{ + /* + * A default refill is not an explicit request, so it must not drop a + * pending out-of-band one, which is applied when @p next runs. + */ + set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); + __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); +} + static void update_curr_scx(struct rq *rq) { struct task_struct *donor; @@ -1662,16 +1672,6 @@ static void dsq_dec_nr(struct scx_dispatch_q *dsq, struct task_struct *p) } } -static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) -{ - /* - * A default refill is not an explicit request, so it must not drop a - * pending out-of-band one, which is applied when @p next runs. - */ - set_task_slice_keep_oob(p, READ_ONCE(sch->slice_dfl)); - __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); -} - /* * Return true if @p is moving due to an internal SCX migration, false * otherwise. From 006e9e9336b96964aafa4bb045e75077ee38f50b Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:37 -1000 Subject: [PATCH 19/34] sched_ext: Fix rescue served time disagreeing with the slice under proxy execution Rescue measures served time as the rescuee's sum_exec_runtime minus a snapshot taken at admission. Under proxy execution that measure follows the wrong task. update_se() credits sum_exec_runtime to rq->curr, the mutex owner that executes, while update_curr_scx() drains the slice of rq->donor and charges its rescue. A rescuee can be either, so its served time and its slice disagree in both directions. As the owner running for another donor, the rescuee accrues served time that drained no slice and charged no budget, and its rescue ends early. As the donor, its rescue runs long: 1. the rescuee blocks on a mutex and the owner runs in its place 2. the owner's run drains the rescuee's slice and the budget, but not the served time 3. the slice runs out with the rescue still open, and scx_rescue_keep() restores a remainder that the owner's run didn't reduce 4. the owner keeps running on the rescue until it releases the mutex, past the admitted slice and overdrawing the budget Count served time in scx_rescue_charge() from the delta that drains the slice. Charging the owner's run to the donor's rescue matches proxy execution, which drains the donor's slice and accounts the cgroup time to the donor for the owner's run. scx_rescue_keep() also ignored the return of scx_set_task_slice(). The call refuses the restore on a rescuee that escalation marked SCX_TASK_PROTECTED, and the rescuee was kept with a zero slice and its protection set. That refusal is now unreachable. Escalation sets the slice to the unserved remainder before it protects the task. From then on every slice write a scheduler can reach is refused. Only a sub-scheduler's task is rescued, and sub-schedulers are cid-form, which rejects direct slice writes. The slice then moves only by what update_curr_scx() drains, and that drain charges the rescue by the same amount, so the slice runs out exactly when the charge ends the rescue. WARN and end the rescue if the restore is refused anyway. Fixes: 5fd501744b10 ("sched_ext: Add bandwidth-limited rescue execution for stranded tasks") Reported-by: Sashiko Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/sub.c | 25 ++++++++++++++++--------- kernel/sched/sched.h | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index a7b14ea16822..998888ca5ea7 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -428,9 +428,7 @@ void scx_init_root_caps(struct scx_sched *sch) /* unserved remainder of @rq's rescuee's admitted slice, 0 once fully served */ static s64 scx_rescue_slice_remaining(struct rq *rq) { - s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap; - - return max(rq->scx.rescue.slice - served, 0); + return max(rq->scx.rescue.slice - rq->scx.rescue.served, 0); } /* @@ -465,6 +463,14 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) lockdep_assert_rq_held(rq); + /* + * Count @delta_exec as served before the clamp below, which bounds only + * the budget charge. update_curr_scx() took the same delta off the + * slice, so a slice set to the unserved remainder, as escalation does, + * reaches zero when the remainder does, see scx_rescue_keep(). + */ + rq->scx.rescue.served += delta_exec; + /* * A rescue slice is bounded by one quantum and tick-driven expiry can * overshoot by up to a tick. Clamp to avoid wild over-charges on VMs. @@ -525,7 +531,9 @@ bool scx_rescue_keep(struct rq *rq, struct task_struct *p) scx_bypassing(scx_task_sched(p), cpu_of(rq))) return false; - scx_set_task_slice(p, remaining); + /* never called on a protected rescuee, so the write can't be refused */ + if (WARN_ON_ONCE(!scx_set_task_slice(p, remaining))) + return false; return true; } @@ -608,10 +616,9 @@ static void scx_rescue_timer_arm(struct rq *rq) * @slice: CPU time to grant * * The schedulers keep their normal control over @p and may preempt or reslice - * it. @slice is measured on served CPU time against the snapshot taken here, so - * neither shortens the rescue, see scx_rescue_charge() and scx_rescue_keep(). - * Prolonged denial escalates into protected execution, see - * scx_rescue_timerfn(). + * it. @slice is measured on served CPU time, so neither shortens the rescue, + * see scx_rescue_charge() and scx_rescue_keep(). Prolonged denial escalates + * into protected execution, see scx_rescue_timerfn(). */ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) { @@ -620,7 +627,7 @@ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) rq->scx.rescue.curr = p; rq->scx.rescue.slice = slice; - rq->scx.rescue.exec_snap = p->se.sum_exec_runtime; + rq->scx.rescue.served = 0; scx_set_task_slice(p, slice); scx_rescue_timer_arm(rq); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 628765910934..dcc639998069 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -816,7 +816,7 @@ struct scx_rq_rescue { u64 clock; /* last budget accrual timestamp */ struct task_struct *curr; /* task being rescued, one at a time */ s64 slice; /* curr's admitted slice */ - u64 exec_snap; /* sum_exec_runtime at admission */ + s64 served; /* curr's served time */ struct timer_list timer; /* paces admission and escalation */ u64 kill_at; /* last ejection, init before any */ }; From 3ce6316379018fc3025b6504ee98a0745ba5b4b5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:38 -1000 Subject: [PATCH 20/34] sched_ext: Restore a rescuee's slice after ops.stopping() put_prev_task_scx() restores a preempted rescuee's unserved slice through scx_rescue_keep() before it calls ops.stopping(). A rescuee whose slice the scheduler zeroes there is left to the reinsert instead: 1. ops.stopping() zeroes the rescuee's slice 2. the reinsert condition special-cases the rescuee and keeps it on the local DSQ with a zero slice 3. the pick's default refill supplies a slice, or the task runs on a zero slice until the tick when its scheduler lacks BASE Move the keep after the stopping call so that scx_rescue_keep() restores the remainder whether a preemption or ops.stopping() zeroed the slice, and drop the special case from the reinsert condition as a rescuee reaching it always has a slice. A rescuee whose slice ops.stopping() zeroed now goes to the tail of the local DSQ with its remainder restored, behind what the scheduler wanted to run first, instead of staying at the head on a zero slice. The slice end in the same block moves with it, which changes nothing for a protected task. Its slice is the unserved remainder, and scx_rescue_charge() ends the protection in the same update_curr_scx() call that zeroes the slice, so the block never sees a protected task in either position. v2: The description no longer says that a protected task's protection outlasts ops.stopping(). Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index d1404a4e1564..b863ab466e9c 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3617,17 +3617,6 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, update_curr_scx(rq); - /* - * If the slice is consumed, protection ends with it. A rescuee - * preempted beforehand keeps going, see scx_rescue_keep(). - */ - if (!p->scx.slice) { - if (unlikely(p == scx_rescuee(rq))) - rescue_keep = scx_rescue_keep(rq, p); - if (!rescue_keep) - scx_task_slice_ended(rq, p); - } - /* * Preserve the running session when proxy execution refreshes the same * donor around an execution-context switch on this rq. @@ -3640,6 +3629,18 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, p->scx.flags &= ~SCX_TASK_RUN_TRACKED; } + /* + * If the slice is consumed, protection ends with it. A rescuee keeps + * going when a preemption or ops.stopping() zeroed its slice, see + * scx_rescue_keep(). + */ + if (!p->scx.slice) { + if (unlikely(p == scx_rescuee(rq))) + rescue_keep = scx_rescue_keep(rq, p); + if (!rescue_keep) + scx_task_slice_ended(rq, p); + } + if (p->scx.flags & SCX_TASK_QUEUED) { if (proxy_block) goto switch_class; @@ -3664,12 +3665,8 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * The exception is an IMMED donor put by proxy_resched_idle(): * the CPU isn't busy, proxy resolution is only retrying through * idle, so keep the donor where it can be picked again. - * - * An open rescue must keep @p on the local DSQ even if the - * scheduler zeroed the slice in ops.stopping() above. */ - if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) && - !scx_bypassing(sch, cpu_of(rq))) { + if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); From 7837315b887f37847151edc4ce03d3a882752bca Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:39 -1000 Subject: [PATCH 21/34] sched_ext: Fix a cpu idling with a runnable rescuee A scheduler preempts a running rescuee like any other task by zeroing its slice. The rescue grant survives the preemption. put_prev_task_scx() restores the unserved remainder through scx_rescue_keep() and queues the rescuee at the tail of the local DSQ, behind whatever the scheduler wanted to run first. That breaks down when the scheduler zeroes the slice but dispatches nothing. dispatch_one() keeps the previous task running only with slice left or, when nothing else is runnable, with its scheduler holding SCX_CAP_BASE on the cpu and not taking the last task through SCX_OPS_ENQ_LAST. A rescuee fails both once its slice is zeroed if its scheduler lacks BASE there or takes the last task: 1. ops.tick() or scx_bpf_task_set_slice() zeroes the rescuee's slice 2. the pick finds nothing else to run and settles on idle 3. put_prev_task_scx() restores the remainder and queues the rescuee on the local DSQ 4. nothing reschedules the cpu, as the pick has settled and __schedule() clears need_resched right after it The cpu idles with the rescuee queued until an unrelated wakeup or the rescue timer's escalation. Restore the remainder from dispatch_one() too. When nothing else is runnable, its last keep now keeps the rescuee running on the remainder, so the pick never settles on idle with a rescuee runnable. put_prev_task_scx() keeps handling a displaced rescuee, where another task won the pick. A rescuee that passes the existing keep-last test is kept there instead and continues as an ordinary task, its rescue ended with the zeroed slice. That is fine as its scheduler holds SCX_CAP_BASE on the cpu. The new keep also covers bypassed tasks run under rescue in the following patches. v2: The dispatch_one() comment and description now say that a rescuee whose scheduler holds SCX_CAP_BASE on the cpu passes the keep-last test and continues as an ordinary task (sashiko). Fixes: 5fd501744b10 ("sched_ext: Add bandwidth-limited rescue execution for stranded tasks") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 38 +++++++++++++++++++++++++++++++++++--- kernel/sched/ext/sub.c | 15 ++++++++------- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index b863ab466e9c..35e2fef66c04 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3397,6 +3397,34 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev verdict = SCX_DSP_PREV; goto has_tasks; } + + /* + * The scheduler preempts a rescuee like any other task by + * zeroing its slice. The grant survives the preemption. The + * rescuee gets the unserved remainder back and runs again + * behind whatever the scheduler wanted to run first. + * + * Two sites restore the remainder. When another task won the + * pick, put_prev_task_scx() restores it and queues the rescuee + * at the tail of the local DSQ. + * + * When nothing else is runnable, the cpu must not go idle with + * the rescuee queued. The keep above takes a rescuee whose + * scheduler holds %SCX_CAP_BASE here and doesn't take the last + * task through %SCX_OPS_ENQ_LAST. + * + * The pick then ends its zeroed slice, and the rescue with it, + * and refills a default slice. The task continues as an + * ordinary one, which is fine as its scheduler holds BASE here. + * + * For any other rescuee, a reinsert from put_prev_task_scx() + * would come after the pick settled on idle. Keep it running on + * the remainder here. + */ + if (unlikely(prev == scx_rescuee(rq)) && scx_rescue_keep(rq, prev)) { + verdict = SCX_DSP_PREV; + goto has_tasks; + } } rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; return SCX_DSP_NONE; @@ -3630,9 +3658,13 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, } /* - * If the slice is consumed, protection ends with it. A rescuee keeps - * going when a preemption or ops.stopping() zeroed its slice, see - * scx_rescue_keep(). + * If the slice is consumed, protection ends with it. + * + * A rescuee keeps going when a preemption or ops.stopping() zeroed its + * slice. Here another task won the pick. scx_rescue_keep() restores the + * unserved remainder and the reinsert below queues the rescuee at the + * tail of the local DSQ. When nothing else is runnable, dispatch_one() + * keeps the rescuee running on the remainder instead. */ if (!p->scx.slice) { if (unlikely(p == scx_rescuee(rq))) diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 998888ca5ea7..5fa9c660aaa6 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -507,16 +507,17 @@ void scx_rescue_end(struct rq *rq) } /** - * scx_rescue_keep - Keep the rescue going for a preempted-out rescuee + * scx_rescue_keep - Keep the rescue going past a zeroed slice * @rq: rq @p is running on - * @p: task under rescue whose slice is exhausted + * @p: task under rescue * - * Called from put_prev_task_scx() to decide what an exhausted slice means for - * the rescuee. scx_rescue_charge() ends the rescue the moment the admitted + * Decide what an exhausted slice means for the rescuee. dispatch_one() calls + * this when nothing else is runnable and put_prev_task_scx() when another task + * won the pick. scx_rescue_charge() ends the rescue the moment the admitted * slice is fully served, so arriving here with the rescue still open means @p - * was preempted. Restore the unserved remainder and return %true - @p stays the - * rescuee and the caller reinserts it at the tail of the local DSQ, behind - * whatever preempted the rescuee. + * was preempted. Restore the unserved remainder and return %true. @p stays the + * rescuee. dispatch_one() keeps it running and put_prev_task_scx() reinserts it + * at the tail of the local DSQ, behind whatever preempted the rescuee. * * Return %false to end the rescue instead - the slice is already fully served, * @p is leaving the rq or bypass is dismantling rescues. From 98889f2cae365fda99b1664151bc7c9908156536 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:40 -1000 Subject: [PATCH 22/34] sched_ext: Mark the rescue grant as a kernel slice with SCX_TASK_KERNEL_SLICE In the following patches, bypassed tasks run under rescue when their host lacks SCX_CAP_BASE on the cpu. Under bypass the kernel zeroes the running task's slice at every tick, as it does not trust the scheduler's slices. A rescue grant has to survive that, or it is served a tick at a time with the rescuee sent to the tail of the local DSQ each time. The generic slice handling can't tell a kernel grant from a scheduler's slice. Mark the grant as a kernel slice with SCX_TASK_KERNEL_SLICE, set at rescue admission and cleared when the slice ends. The generic slice handling leaves it alone. The default refill does nothing for it, and the three zeroings go through the new expire_task_slice(), which skips it: - the bypass tick's zeroing of the running task - the eviction of a running task that can't stay on the cpu when bypass lifts - the cap-revoke eviction of a running task Schedulers keep their control. Preemption and scx_bpf_task_set_slice() still apply, and the rescue stays bounded by served time. The sites that recognized the rescuee by identity, the reenqueue scan and put_prev_task_scx()'s reinsert, test the flag instead. scx_task_has_kernel_slice() hides the test behind the scx_has_subs() static key, as only a sub-scheduler's task can carry the flag. The flag's entries join the enum tables in tools/sched_ext. No kernel slice reaches a zeroing until bypassed tasks run under rescue. The cap-revoke eviction skipped the rescuee already, and bypass entry ends an open rescue, so the bypass tick and the eviction at bypass lift see none. The refill is the one behavior change. The pick refills a zero slice only for a task that can stay on the cpu, so it concerns a rescuee whose scheduler holds BASE there. A scheduler may zero such a queued rescuee's slice, and the next pick used to refill it to the default. Now the rescuee runs on the zero slice until the next tick. There put_prev_task_scx() restores the remainder when another task is runnable. When nothing else is runnable, dispatch_one() keeps the rescuee as the last task. The pick then ends its zero slice, which ends the rescue, and refills the default. A scheduler with SCX_OPS_ENQ_LAST skips the keep-last test. scx_rescue_keep() restores the remainder there instead and the rescue goes on. v2: - scx_rescue_admit()'s kerneldoc no longer mentions a served-time snapshot. - The description covers a scheduler with SCX_OPS_ENQ_LAST, whose rescuee dispatch_one() keeps on the remainder. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 11 +++++ kernel/sched/ext/ext.c | 45 ++++++++++++++----- kernel/sched/ext/sub.c | 10 +++-- kernel/sched/ext/sub.h | 7 +++ .../sched_ext/include/scx/enum_defs.autogen.h | 1 + .../sched_ext/include/scx/enums_abi.autogen.h | 1 + 6 files changed, 60 insertions(+), 15 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index b341c328948e..fcf85b87fdd4 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -106,6 +106,17 @@ enum scx_ent_flags { SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */ + /* + * The kernel owns the slice, currently the rescue grant. While set: + * + * - refill_task_slice_dfl() and expire_task_slice() leave the slice + * alone. + * - Reenqueue scans skip the task. + * - put_prev_task_scx() keeps a preempted task on the local DSQ. + * - Scheduler slice writes and preemption still apply. + */ + SCX_TASK_KERNEL_SLICE = 1 << 8, /* slice owned by the kernel */ + /* * The multi-bit fields are packed at the top, below SCX_TASK_CURSOR, so * that the single-bit flags stay contiguous from bit 0. diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 35e2fef66c04..01b0fbf51722 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1472,8 +1472,8 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice) * @rq: rq @p is on * @p: task of interest * - * End what rides on the slice - the protection, and the rescue if @p is being - * rescued. + * End what rides on the slice - the protection, the kernel's ownership of the + * slice, and the rescue if @p is being rescued. * * A dequeue normally ends the slice too. The exception is a save/restore pair * on the running task. Attribute changes like renice cycle the task through @@ -1485,7 +1485,7 @@ void scx_task_slice_ended(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); - p->scx.flags &= ~SCX_TASK_PROTECTED; + p->scx.flags &= ~(SCX_TASK_PROTECTED | SCX_TASK_KERNEL_SLICE); if (unlikely(p == scx_rescuee(rq))) scx_rescue_end(rq); } @@ -1569,8 +1569,16 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e p->scx.dsq_vtime = vtime; } +/* + * refill_task_slice_dfl() supplies a slice when the scheduler left none and + * expire_task_slice() takes it away to force the task off the cpu. Neither + * touches a kernel slice, see %SCX_TASK_KERNEL_SLICE. + */ static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) { + if (scx_task_has_kernel_slice(p)) + return; + /* * A default refill is not an explicit request, so it must not drop a * pending out-of-band one, which is applied when @p next runs. @@ -1579,6 +1587,17 @@ static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); } +static void expire_task_slice(struct task_struct *p) +{ + /* + * Unlike the refill above, an expiry supersedes a pending out-of-band + * request. A kept request would be applied at the next + * update_curr_scx() and undo the expiry. + */ + if (!scx_task_has_kernel_slice(p)) + scx_set_task_slice(p, 0); +} + static void update_curr_scx(struct rq *rq) { struct task_struct *donor; @@ -3706,11 +3725,15 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, u64 enq_flags = 0; /* - * Keep a preempted rescue going. If preempted - * by another SCX task, append to the local DSQ, - * see scx_rescue_keep(). + * The enqueue caps gate the scheduler's + * placement. The kernel placed a kernel-slice + * task here itself, so reinsert it without a + * cap check, at the head like any preempted + * task. When scx_rescue_keep() restored a slice + * the scheduler zeroed, the scheduler wanted + * something else to run first, so append. */ - if (unlikely(p == scx_rescuee(rq))) { + if (unlikely(scx_task_has_kernel_slice(p))) { enq_flags |= SCX_ENQ_IGNORE_CAPS; if (!rescue_keep) enq_flags |= SCX_ENQ_HEAD; @@ -4417,7 +4440,7 @@ static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued) * management. */ if (scx_bypassing(sch, cpu_of(rq))) - scx_set_task_slice(donor, 0); + expire_task_slice(donor); else if (SCX_HAS_OP(sch, tick)) SCX_CALL_OP_TASK(sch, tick, rq, donor); @@ -5040,7 +5063,7 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; - if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq))) + if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || scx_task_has_kernel_slice(p))) return false; *reason = SCX_TASK_REENQ_KFUNC; @@ -5143,7 +5166,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && rq->donor->sched_class == &ext_sched_class && scx_task_reenq_on_cap_revoke(rq, rq->donor)) { - scx_set_task_slice(rq->donor, 0); + expire_task_slice(rq->donor); resched_curr(rq); } @@ -6999,7 +7022,7 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (bypass) scx_task_slice_ended(rq, p); else if (!scx_task_can_stay_on_cpu(rq, p)) - scx_set_task_slice(p, 0); + expire_task_slice(p); } /* diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 5fa9c660aaa6..139257f3411f 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -616,10 +616,11 @@ static void scx_rescue_timer_arm(struct rq *rq) * @p: task being admitted, off any DSQ * @slice: CPU time to grant * - * The schedulers keep their normal control over @p and may preempt or reslice - * it. @slice is measured on served CPU time, so neither shortens the rescue, - * see scx_rescue_charge() and scx_rescue_keep(). Prolonged denial escalates - * into protected execution, see scx_rescue_timerfn(). + * The slice is a kernel slice, see %SCX_TASK_KERNEL_SLICE. The schedulers keep + * their normal control over @p and may preempt or reslice it. @slice is + * measured on served CPU time, so neither shortens the rescue, see + * scx_rescue_charge() and scx_rescue_keep(). Prolonged denial escalates into + * protected execution, see scx_rescue_timerfn(). */ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) { @@ -630,6 +631,7 @@ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) rq->scx.rescue.slice = slice; rq->scx.rescue.served = 0; scx_set_task_slice(p, slice); + p->scx.flags |= SCX_TASK_KERNEL_SLICE; scx_rescue_timer_arm(rq); } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index eb1dba6d51e2..de80a8af524c 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -254,12 +254,19 @@ static inline struct task_struct *scx_rescuee(struct rq *rq) return rq->scx.rescue.curr; } +/* does the kernel own @p's slice? see %SCX_TASK_KERNEL_SLICE */ +static inline bool scx_task_has_kernel_slice(const struct task_struct *p) +{ + return scx_has_subs() && (p->scx.flags & SCX_TASK_KERNEL_SLICE); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; } static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; } +static inline bool scx_task_has_kernel_slice(const struct task_struct *p) { return false; } #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 4fa82033b99c..362e8bb34499 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -108,6 +108,7 @@ #define HAVE_SCX_TASK_IMMED #define HAVE_SCX_TASK_PROTECTED #define HAVE_SCX_TASK_RUN_TRACKED +#define HAVE_SCX_TASK_KERNEL_SLICE #define HAVE_SCX_TASK_STATE_SHIFT #define HAVE_SCX_TASK_STATE_BITS #define HAVE_SCX_TASK_STATE_MASK diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index 05a460611c4d..ad8f56f581ad 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -120,6 +120,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, { "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU }, + { "scx_ent_flags", "SCX_TASK_KERNEL_SLICE", 0x100LLU }, { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x19LLU }, { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0xe000000LLU }, From 8197fa420ddd6cd9d3a023b63db34b798d163cc1 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:41 -1000 Subject: [PATCH 23/34] sched_ext: Refuse extending a kernel slice A rescuee can keep running after its grant is served, on a cpu its scheduler has lost. The write rules deny a slice extension only where the scheduler lacks SCX_CAP_BASE on the cpu, and a scheduler holding BASE but not ENQ can have a task under rescue there: 1. the scheduler extends the rescuee's slice 2. BASE is revoked. The cap-revoke eviction skips the rescuee. 3. the grant is served and scx_rescue_charge() ends the rescue 4. the task runs out the extension on the cpu its scheduler lost, outside the rescue's bandwidth A scheduler shortens or zeroes a rescuee's slice to preempt it, and scx_rescue_keep() resumes the grant afterwards. An extension serves no such purpose. Refuse extending a kernel slice in both scheduler write paths, the direct write in scx_bpf_task_set_slice() and the out-of-band application, counted in SCX_EV_SLICE_DENIED like an extension without BASE. Shortening stays allowed. scx_rescue_keep()'s restore of the remainder raises the slice from zero. Like the kernel's other slice writes, it goes through scx_set_task_slice() and not the scheduler write paths. v2: - scx_rescue_admit()'s kerneldoc no longer mentions a served-time snapshot. - SCX_EV_SLICE_DENIED's comment lists every refusal it counts. - scx_bpf_task_set_slice()'s kerneldoc says that shortening is refused on a protected slice. Fixes: 5fd501744b10 ("sched_ext: Add bandwidth-limited rescue execution for stranded tasks") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- include/linux/sched/ext.h | 2 +- kernel/sched/ext/ext.c | 30 +++++++++++++++++++----------- kernel/sched/ext/internal.h | 6 ++++-- kernel/sched/ext/sub.c | 10 +++++----- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index fcf85b87fdd4..f3564581c317 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -113,7 +113,7 @@ enum scx_ent_flags { * alone. * - Reenqueue scans skip the task. * - put_prev_task_scx() keeps a preempted task on the local DSQ. - * - Scheduler slice writes and preemption still apply. + * - Schedulers may shorten the slice, never extend it. */ SCX_TASK_KERNEL_SLICE = 1 << 8, /* slice owned by the kernel */ diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 01b0fbf51722..90b3df2cdbbe 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1385,7 +1385,8 @@ enum scx_slice_oob_consts { * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration. * * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is - * refused. See set_task_slice_keep_oob(). + * refused. See set_task_slice_keep_oob(). A kernel slice, see + * %SCX_TASK_KERNEL_SLICE, can be shortened but not extended. * * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct @@ -1511,9 +1512,9 @@ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 /* * Apply a pending out-of-band slice request under @rq's lock. A request whose * packed id no longer matches @p's current owner is dropped. An extension needs - * baseline cpu access on @p's cid, shortening is always allowed, and a - * protected slice refuses both. %SCX_EV_SLICE_DENIED counts the denials. See - * the write rules above. + * baseline cpu access on @p's cid and is refused on a kernel slice. Shortening + * is always allowed. A protected slice refuses both. %SCX_EV_SLICE_DENIED + * counts the denials. See the write rules above. */ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) { @@ -1537,7 +1538,8 @@ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p) slice = dur == SCX_SLICE_OOB_DUR_MASK ? SCX_SLICE_INF : dur; if (slice > p->scx.slice && - unlikely(scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) { + unlikely(scx_task_has_kernel_slice(p) || + scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE))) { __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1); return; } @@ -10427,10 +10429,12 @@ __bpf_kfunc_start_defs(); * rules above for the ownership model. * * Raising the slice is honored only while the scheduler holds %SCX_CAP_BASE on - * @p's cpu, otherwise it is counted in %SCX_EV_SLICE_DENIED. Shortening is - * always allowed. On the stashed path the slice is packed into an atomic64_t - * with the scheduler id and a flag bit, so a slice too large to fit is clamped - * and counted in %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved. + * @p's cpu, and never on a kernel slice, see %SCX_TASK_KERNEL_SLICE. Shortening + * is honored unless @p is protected, see %SCX_TASK_PROTECTED, which refuses + * every write. A refused write is counted in %SCX_EV_SLICE_DENIED. On the + * stashed path the slice is packed into an atomic64_t with the scheduler id and + * a flag bit, so a slice too large to fit is clamped and counted in + * %SCX_EV_SLICE_CLAMPED. %SCX_SLICE_INF is preserved. * * Return %true on success, %false if @p is not on the calling scheduler. */ @@ -10468,9 +10472,13 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, return true; } - /* under the rq lock: apply now, extensions gated on baseline access */ + /* + * @p's rq is locked, so apply now. An extension needs baseline access + * and is refused on a kernel slice. + */ if (slice > p->scx.slice && - unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { + unlikely(scx_task_has_kernel_slice(p) || + scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) { __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1); return true; } diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 752a9a2a94b0..ed63e29c9442 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1343,8 +1343,10 @@ struct scx_event_stats { s64 SCX_EV_SLICE_CLAMPED; /* - * The number of times a slice extension was denied because the - * scheduler lacked baseline cpu access on the task's cpu. + * The number of times a slice write was refused. A raise is refused + * without baseline cpu access on the task's cpu and on a kernel slice. + * Every write is refused on a protected slice, a preemption's zeroing + * included. */ s64 SCX_EV_SLICE_DENIED; diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 139257f3411f..6fa8cea1ef88 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -616,11 +616,11 @@ static void scx_rescue_timer_arm(struct rq *rq) * @p: task being admitted, off any DSQ * @slice: CPU time to grant * - * The slice is a kernel slice, see %SCX_TASK_KERNEL_SLICE. The schedulers keep - * their normal control over @p and may preempt or reslice it. @slice is - * measured on served CPU time, so neither shortens the rescue, see - * scx_rescue_charge() and scx_rescue_keep(). Prolonged denial escalates into - * protected execution, see scx_rescue_timerfn(). + * The slice is a kernel slice, see %SCX_TASK_KERNEL_SLICE. The schedulers may + * preempt @p or shorten its slice, never extend it. @slice is measured on + * served CPU time, so neither shortens the rescue, see scx_rescue_charge() and + * scx_rescue_keep(). Prolonged denial escalates into protected execution, see + * scx_rescue_timerfn(). */ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) { From 20a61e41423374fb4ece816528fe3f0f9faf820a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:42 -1000 Subject: [PATCH 24/34] sched_ext: Fix a cpu idling after put_prev_task_scx() queues its task on the local DSQ put_prev_task_scx() enqueues a still-runnable task when the pick has settled on idle. The insert can land the task on the cpu's own local DSQ, and then the cpu must not go idle over it. 7a9df7667963 ("sched_ext: Fix stall when a task enqueued with SCX_ENQ_LAST lands back on its CPU's local DSQ") reschedules the idle task in that case, but only after the SCX_ENQ_LAST insert. The plain insert, taken when the task can't stay on the cpu, can land on the local DSQ too: rescue can admit a task there when its scheduler lacks SCX_CAP_BASE on the cpu. Without the resched the cpu idles with the task queued until an unrelated wakeup or the rescue timer's escalation. Reschedule after either insert. In the following patches, the plain insert becomes the common path for bypassed tasks under rescue. v2: The description's forward reference to the plain insert becoming the common path no longer names the following patch. Fixes: 5fd501744b10 ("sched_ext: Add bandwidth-limited rescue execution for stranded tasks") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 90b3df2cdbbe..27b027833f06 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3658,6 +3658,8 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, (rq->scx.flags & SCX_RQ_PROXY_PENDING); bool proxy_block = rq->scx.flags & SCX_RQ_PROXY_BLOCKING; bool rescue_keep = false; + bool going_idle = !proxy_put && next && + sched_class_above(&ext_sched_class, next->sched_class); rq->scx.flags &= ~SCX_RQ_PROXY_PENDING; @@ -3763,25 +3765,28 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * locally runnable and can legitimately go idle with @p still * runnable (see do_pick_task_scx()). */ - if (!proxy_put && next && - sched_class_above(&ext_sched_class, next->sched_class) && - scx_task_can_stay_on_cpu(rq, p)) { + if (going_idle && scx_task_can_stay_on_cpu(rq, p)) { WARN_ON_ONCE(!sched_core_enabled(rq) && !(sch->ops.flags & SCX_OPS_ENQ_LAST)); scx_do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); - - /* - * A task put on the local DSQ runs, as anywhere else. - * Here the insert can't reschedule on its own: the pick - * has settled on @next and @p is still curr, so - * resched_curr() would flag @p and __schedule() clears - * that right after the pick. Flag @next instead. - */ - if (p->scx.dsq == &rq->scx.local_dsq) - set_tsk_need_resched(next); } else { scx_do_enqueue_task(rq, p, 0, -1); } + + /* + * @p may be on the local DSQ now. With %SCX_ENQ_LAST the + * scheduler can put the last runnable task there. The plain + * insert runs when @p cannot stay on this cpu, and rescue can + * still admit it here. + * + * The cpu must not go idle with @p queued on its local DSQ. The + * usual resched doesn't work: the pick has settled on @next and + * @p is still curr, so resched_curr() would set need_resched on + * @p, which __schedule() clears right after the pick. Set it on + * @next instead. + */ + if (going_idle && p->scx.dsq == &rq->scx.local_dsq) + set_tsk_need_resched(next); } switch_class: From b9843b0399e807824020ccf7a47d06c19fbfa2d2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:43 -1000 Subject: [PATCH 25/34] sched_ext: Expire prev's slice in dispatch_one() under bypass The next patch keeps a bypassed rescuee on the local DSQ and needs dispatch_one() to keep it running too. dispatch_one()'s keep test refuses any prev whose scheduler is bypassing, so another local task could run ahead of the rescuee, even an escalated SCX_TASK_PROTECTED one. The test exists because a cpu-form scheduler can write p->scx.slice directly from BPF, so the slice can't be trusted under bypass. The refusal doesn't hold anyway. Since 025b1bd41965 ("sched_ext: Implement hierarchical bypass mode"), the host's ops.dispatch() runs while prev's scheduler bypasses, and the post-dispatch test in scx_dispatch_sched() keeps a queued prev with slice left without checking bypass. A task dispatched in between waits behind prev until the next bypass tick zeroes the slice, at most a tick. Expire the slice before the keep test instead when prev's scheduler is bypassing, as the bypass tick would. Neither test keeps prev on a zeroed slice, so prev rotates. A kernel slice survives the expiry and keeps prev. Only a sub-scheduler's task can hold one, and sub-schedulers are cid-form, which rejects direct slice writes, so it can be trusted. Drop the stale pointer to scx_disable_workfn(), update scx_bypass()'s kerneldoc, and add a TODO noting that the bypass slice handling can be cleaned up once the cpu form is deprecated and the kfuncs refuse slice raises from a bypassing scheduler. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 27b027833f06..f1c833f1b78d 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3376,18 +3376,33 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev if (prev->sched_class == &ext_sched_class) { update_curr_scx(rq); + /* + * Under bypass, @prev's slice can't be trusted as a cpu-form + * scheduler can write it directly from BPF. Skipping the keep + * below is not enough as the post-dispatch test in + * scx_dispatch_sched() would keep @prev on the same slice. + * + * Expire it as the bypass tick would, so that neither test + * keeps @prev on it and @prev rotates through the bypass + * enqueue. A kernel slice is left alone and the keep below + * keeps @prev on it, see %SCX_TASK_KERNEL_SLICE. + * + * TODO: Once the cpu form is deprecated and the kfuncs refuse + * slice raises from a bypassing scheduler, slices can be + * trusted under bypass. This expiry, the bypass tick's zeroing + * and the related slice handling can then be cleaned up. + */ + if (scx_bypassing(scx_task_sched(prev), cpu)) + expire_task_slice(prev); + /* * If @prev is runnable & has slice left, it has priority and * fetching more just increases latency for the fetched tasks. * Tell pick_task_scx() to keep running @prev. If the BPF * scheduler wants to handle this explicitly, it should * implement ->cpu_release(). - * - * See scx_disable_workfn() for the explanation on the bypassing - * test. */ - if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice && - !scx_bypassing(scx_task_sched(prev), cpu)) { + if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) { verdict = SCX_DSP_PREV; goto has_tasks; } @@ -6917,9 +6932,10 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called. * The bypassed tasks are on the bypass DSQs, out of their reach. * - * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice - * can't be trusted. Whenever a tick triggers, the running task is rotated to - * the tail of the queue. + * - dispatch_one() expires prev's slice before its keep test as the slice can't + * be trusted. Each tick does the same and rotates the running task to the + * tail of the queue. A kernel slice is left alone, see + * %SCX_TASK_KERNEL_SLICE. * * - pick_next_task() suppresses zero slice warning. * From 947dd3eb80df8e83683182271d2782867befd333 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:44 -1000 Subject: [PATCH 26/34] sched_ext: Fix bypassed tasks stalling when the host lacks SCX_CAP_BASE A bypassed task stalls on a cpu where its host lacks SCX_CAP_BASE. The host, the nearest ancestor that is not bypassing, runs a bypassing sub-scheduler's tasks through its per-cpu bypass DSQs. A cpu's dispatch only descends into schedulers holding BASE on it, so the host's bypass DSQ on such a cpu is never consumed until the host regains the cap. Resolve a bypassed task against its host at enqueue time. If the host lacks BASE on the cpu, send the task to rescue on that cpu, which runs it at bounded bandwidth without the host's dispatch having to reach it. The enqueue does not look above the host for an ancestor that holds BASE. That ancestor would run the task on cpus it never granted the host. With rescue disabled the task goes to the host's bypass DSQ as before and stalls there until the host regains BASE or the bypass ends, or until the stall watchdog ejects the host, which it blames for a task stuck on its DSQ. A task that must run on the cpu, migration-disabled or with a migration pending, is admitted to the local DSQ instead, as any insert of such a task is. Bypass also used to end a rescue, sending a preempted rescuee to the bypass DSQ with every other preempted task. Now that rescue is how a bypassed task runs while its host lacks BASE, keep a preempted rescuee on the local DSQ under bypass too. dispatch_one() also keeps such a rescuee running, as an earlier patch's bypass expiry leaves a kernel slice alone. The grant stays bounded by served time. A task scx_fail_parent() punted to its bypassing parent at NONE can be a rescuee too. switched_from_scx() ends its slice as it does for an enabled task, so that a NONE rescuee leaving sched_ext while running doesn't leave rq->scx.rescue.curr pointing at it. While at it, the rescue test in __scx_resolve_local_dsq() goes through the new scx_rescue_enabled() helper. v2: - The description notes that dispatch_one() keeps a bypassed rescuee too, through the new patch before this one (sashiko). - scx_rescue_keep()'s kerneldoc drops dispatch_one() declining a rescuee with slice left, which the new patch removes. v3: - With rescue disabled, a task that must run on the cpu is admitted to the local DSQ instead of waiting on the host's bypass DSQ. - The description names the stall watchdog as the way out for a task stuck without rescue. v4: - switched_from_scx() ends the slice of a NONE task, which can now be a rescuee (sashiko). - scx_punt_task()'s kerneldoc no longer says that switched_from_scx() skips a NONE task. - scx_bypass()'s kerneldoc and the ops.sub_ecaps_updated() doc say where a bypassed task can be besides the bypass DSQs. Fixes: 147d1885f390 ("sched_ext: Add the SCX_CAP_ENQ_IMMED cap") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 50 +++++++++++++++++++++++++++++++------ kernel/sched/ext/internal.h | 5 ++-- kernel/sched/ext/sub.c | 24 ++++++++++-------- kernel/sched/ext/sub.h | 9 +++++++ 4 files changed, 67 insertions(+), 21 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f1c833f1b78d..3645a844cfbc 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -388,8 +388,12 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) return __setscheduler_class(p->policy, p->prio); } -static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) +static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { + struct scx_sched *host = scx_exec_sched(sch, cpu_of(rq)); + struct scx_dispatch_q *dsq; + /* * If @sch is a sub-sched which is bypassing, its tasks should go into * the bypass DSQs of the nearest ancestor which is not bypassing. The @@ -400,8 +404,29 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c * Whenever a sched starts bypassing, all runnable tasks in its subtree * are re-enqueued after scx_bypassing() is turned on, guaranteeing that * all tasks are transferred to the right DSQs. + * + * A cpu's dispatch only descends into a host holding %SCX_CAP_BASE on + * it, so the bypass DSQ of a host without BASE here is never consumed. + * Send @p to rescue on this cpu instead, resolving against @host. + * scx_dispatch_enqueue() resolves a local DSQ again, against the + * bypassing @sch. %SCX_ENQ_IGNORE_CAPS makes that second resolution + * admit @p. + * + * A migration-disabled task can't leave this cpu, and a task with a + * migration pending must run here until the migration moves it. Either + * would wait on the unconsumed bypass DSQ until the host regains BASE. + * Resolve them too when rescue is disabled. The resolution admits both + * to the local DSQ, as it does any insert of such a task. */ - return scx_bypass_dsq(scx_exec_sched(sch, cpu), cpu); + if (unlikely(scx_missing_caps(host, cpu_of(rq), SCX_CAP_BASE)) && + (scx_rescue_enabled() || is_migration_disabled(p) || p->migration_pending)) { + *enq_flags |= SCX_ENQ_RESCUE; + dsq = scx_resolve_local_dsq(host, rq, p, enq_flags); + *enq_flags |= SCX_ENQ_IGNORE_CAPS; + } else { + dsq = scx_bypass_dsq(host, cpu_of(rq)); + } + return dsq; } /** @@ -2353,7 +2378,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, dsq = find_global_dsq(sch, task_cpu(p)); goto enqueue; bypass: - dsq = bypass_enq_target_dsq(sch, task_cpu(p)); + dsq = bypass_enq_target_dsq(sch, rq, p, &enq_flags); goto enqueue; enqueue: @@ -3735,8 +3760,13 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * The exception is an IMMED donor put by proxy_resched_idle(): * the CPU isn't busy, proxy resolution is only retrying through * idle, so keep the donor where it can be picked again. + * + * A task with a kernel slice stays on the local DSQ even while + * @sch bypasses. Under bypass, rescue runs @p when the host + * lacks %SCX_CAP_BASE, see bypass_enq_target_dsq(). */ - if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { + if (p->scx.slice && (!scx_bypassing(sch, cpu_of(rq)) || + scx_task_has_kernel_slice(p))) { if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); @@ -4987,11 +5017,14 @@ static void switched_from_scx(struct rq *rq, struct task_struct *p) * parent's own teardown). There is nothing to disable; calling * scx_disable_task() would WARN on the non-%ENABLED state and trigger a * NONE -> READY validation failure. + * + * @p can still be a rescuee under the parent. End its slice as + * scx_disable_task() does for an enabled task. */ if (scx_get_task_state(p) == SCX_TASK_NONE) - return; - - scx_disable_task(scx_task_sched(p), p); + scx_task_slice_ended(rq, p); + else + scx_disable_task(scx_task_sched(p), p); } static void switched_to_scx(struct rq *rq, struct task_struct *p) {} @@ -6930,7 +6963,8 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * - ops.dispatch() is ignored. * * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called. - * The bypassed tasks are on the bypass DSQs, out of their reach. + * The bypassed tasks are on the bypass DSQs, under rescue or admitted to a + * local DSQ, out of their reach. * * - dispatch_one() expires prev's slice before its keep test as the slice can't * be trusted. Each tick does the same and rotates the running task to the diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index ed63e29c9442..ecd677c4b561 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -924,8 +924,9 @@ struct sched_ext_ops { * ops.sub_attach() and any change since are reported then. A cpu that * isn't online at that point reports them once it comes online. * Deliveries continue while this sched bypasses. Its tasks are then on - * the bypass DSQs, where this op can't insert or move them. Deliveries - * stop once this sched is being disabled. + * the bypass DSQs, under rescue or admitted to a local DSQ, where this + * op can't insert or move them. Deliveries stop once this sched is + * being disabled. * * Until this sched is being disabled, whenever an op of this sched runs * under a cid's rq lock, its effective caps on the cid are the @after diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 6fa8cea1ef88..01215384eaa5 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -36,7 +36,7 @@ static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled); static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */ /* latched at root enable before any rescue runs */ -static s32 scx_rescue_bw_1024; +s32 scx_rescue_bw_1024; /* 0 if disabled, see scx_rescue_enabled() */ static s64 scx_rescue_quantum_ns; static s64 scx_rescue_sat_delta_ns; static unsigned long scx_rescue_decay_halflife; @@ -519,8 +519,8 @@ void scx_rescue_end(struct rq *rq) * rescuee. dispatch_one() keeps it running and put_prev_task_scx() reinserts it * at the tail of the local DSQ, behind whatever preempted the rescuee. * - * Return %false to end the rescue instead - the slice is already fully served, - * @p is leaving the rq or bypass is dismantling rescues. + * Return %false to end the rescue instead - the slice is already fully served + * or @p is leaving the rq. */ bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { @@ -528,8 +528,7 @@ bool scx_rescue_keep(struct rq *rq, struct task_struct *p) lockdep_assert_rq_held(rq); - if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED) || - scx_bypassing(scx_task_sched(p), cpu_of(rq))) + if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED)) return false; /* never called on a protected rescuee, so the write can't be refused */ @@ -887,9 +886,11 @@ void scx_rescue_init(struct rq *rq) * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled * admission. * - * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued - * to and run by its nearest non-bypassing ancestor. If root is bypassing, it - * always holds all caps. + * bypass_enq_target_dsq() also calls this with the host as @sch when the host + * lacks %SCX_CAP_BASE on @rq's cid, if rescue is enabled or @p must run on the + * cpu, migration-disabled or with a migration pending. The bypassed task is + * then admitted or diverted like a capless %SCX_ENQ_RESCUE insert by the host + * itself. */ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) @@ -932,7 +933,7 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq scx_divert_strip_flags(p, enq_flags); /* the enqueuer opted for rescue instead of rejection and reenqueue */ - if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { + if (scx_rescue_enabled() && (*enq_flags & SCX_ENQ_RESCUE)) { __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1); if (scx_rescue_try_admit(rq, p)) return &rq->scx.local_dsq; @@ -1656,8 +1657,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) * * Used when @to must take over @p but failed to initialize it. Bypass keeps * scheduling decisions away from @to but @p can still trigger its task ops, - * which may confuse the BPF side. @to is dying anyway. The exit paths skip - * %NONE tasks (see __scx_disable_and_exit_task() and switched_from_scx()). + * which may confuse the BPF side. @to is dying anyway. The exit paths call no + * op on a %NONE task. __scx_disable_and_exit_task() skips it and + * switched_from_scx() only ends its slice. */ static void scx_punt_task(struct scx_sched *to, struct task_struct *p) { diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index de80a8af524c..b6d4a522a5a3 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -14,6 +14,8 @@ #ifdef CONFIG_EXT_SUB_SCHED +extern s32 scx_rescue_bw_1024; + struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch); @@ -51,6 +53,12 @@ void scx_rescue_dump(struct seq_buf *s, struct rq *rq); void scx_rescue_set_knobs(struct scx_sched *sch); void scx_rescue_init(struct rq *rq); +/* false only when the root disabled rescue, see scx_rescue_set_knobs() */ +static inline bool scx_rescue_enabled(void) +{ + return likely(scx_rescue_bw_1024); +} + /* * cgrp->scx_sched is written by root/sub enable/disable under all of * scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the @@ -128,6 +136,7 @@ static inline void scx_rescue_end(struct rq *rq) {} static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } static inline void scx_rescue_flush(struct rq *rq) {} static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {} +static inline bool scx_rescue_enabled(void) { return false; } static inline void scx_rescue_set_knobs(struct scx_sched *sch) {} static inline void scx_rescue_init(struct rq *rq) {} static inline void scx_dec_has_subs(struct scx_sched *sch) {} From 53cc8dd06944979fd6a4754958aa8d6fa159f2da Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:45 -1000 Subject: [PATCH 27/34] sched_ext: Fix the bypass load balancer stranding tasks on cpus where the host lacks SCX_CAP_BASE The bypass load balancer spreads a host's bypassed tasks over the cpus of a node by moving them between its per-cpu bypass DSQs. It counts every cpu of the node as capacity and takes any cpu below the target as a donee. A cpu's dispatch only descends into schedulers holding SCX_CAP_BASE on it, so a bypass DSQ on a cpu where the host lacks BASE is never consumed. A task moved there sits until the host regains the cap or the bypass window ends. Count only cpus where the host holds BASE as capacity and pick donees from them. The target rises to spread the node's tasks over the cpus that can run them, and a capless cpu receives nothing. A capless cpu with tasks is still an ordinary donor, offloaded down to the donor target when its queue grows past the threshold; draining it is left to the following patches. Fixes: 147d1885f390 ("sched_ext: Add the SCX_CAP_ENQ_IMMED cap") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3645a844cfbc..4ee802fad9df 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6741,7 +6741,9 @@ static void bypass_lb_node(struct scx_sched *sch, int node) u32 nr = READ_ONCE(scx_bypass_dsq(sch, cpu)->nr); nr_tasks += nr; - nr_cpus++; + /* only a cpu whose dispatch consumes its DSQ is capacity */ + if (!scx_missing_caps(sch, cpu, SCX_CAP_BASE)) + nr_cpus++; before_min = min(nr, before_min); before_max = max(nr, before_max); @@ -6760,7 +6762,9 @@ static void bypass_lb_node(struct scx_sched *sch, int node) cpumask_clear(donee_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target) + /* a donee's DSQ is consumed only if @sch holds %SCX_CAP_BASE */ + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) < nr_target && + !scx_missing_caps(sch, cpu, SCX_CAP_BASE)) cpumask_set_cpu(cpu, donee_mask); } From f0bbe21d73e2784fea25524af6e8e241b40b5b22 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:46 -1000 Subject: [PATCH 28/34] sched_ext: Factor out bypass_lb_move() and insert with the donor DSQ unlocked Prepare the bypass load balancer for moving tasks homed on other cpus. Such a move takes the task's own rq lock through the remote consumer's handshake, which drops the donor DSQ lock on the way, so its insert runs with the donor DSQ unlocked and under the task's rq lock. The balancer's own move holds the donor DSQ lock throughout, with the donee DSQ lock nested inside. Switch the donor-homed move to the same scheme so that one body can serve both: factor it out of bypass_lb_cpu() into bypass_lb_move(), which takes the task off the donor DSQ and inserts with the DSQ unlocked. A task that can't run on the donee goes back to the head of the donor DSQ instead of being skipped in place. The walk parks its cursor behind the task before each move and resumes from it afterwards. It is bounded to the tasks queued when it began, by their DSQ sequence numbers, so a task it put back is not visited again in the same pass. Before, the walk held both the donor rq lock and the donor DSQ lock across every move. The donor DSQ can be very long and contended, a walk competing with the cpu's dispatch and the enqueues for the DSQ lock would crawl, and the walk is what relieves that contention by moving tasks off the DSQ. The walk now cycles the DSQ lock around each insert but still holds the rq lock throughout, and that alone keeps dispatch and enqueues away. The DSQ lock is uncontended when retaken, so cycling it is cheap. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 131 ++++++++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 40 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 4ee802fad9df..cddfcc61c454 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6619,6 +6619,57 @@ bool scx_hardlockup(int cpu) return true; } +/** + * bypass_lb_move - Move one task off a bypass DSQ for the balancer + * @sch: host sched whose bypass DSQs are balanced + * @donor_rq: rq of the cpu whose bypass DSQ gives up @p, locked + * @donor_dsq: @sch's bypass DSQ on @donor_rq's cpu, locked + * @p: task on @donor_dsq, homed on @donor_rq + * @donee: cpu whose bypass DSQ receives @p + * + * Take @p off @donor_dsq and insert it into the bypass DSQ of @donee. The + * caller picks @donee from @p's affinity alone and task_can_run_on_remote_rq() + * can still refuse the move under @p's rq lock. @p then goes back to the head + * of @donor_dsq. + * + * @donor_rq and @donor_dsq are locked on entry and on return. @donor_dsq is + * unlocked across the insert, so the caller's walk must find its place in it + * again after the call. + * + * Return the cpu that received @p, or -1 if @p went back onto @donor_dsq. + */ +static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, + struct scx_dispatch_q *donor_dsq, struct task_struct *p, s32 donee) +{ + struct scx_dispatch_q *dst_dsq; + u64 enq_flags; + s32 ret; + + lockdep_assert_rq_held(donor_rq); + lockdep_assert_held(&donor_dsq->lock); + + scx_wait_dispatching(p); + dispatch_dequeue_locked(p, donor_dsq); + raw_spin_unlock(&donor_dsq->lock); + + /* @p's affinity is stable under its rq lock */ + if (task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) { + dst_dsq = scx_bypass_dsq(sch, donee); + enq_flags = 0; + ret = donee; + } else { + /* back where it was, at the head so that the walk skips it */ + dst_dsq = donor_dsq; + enq_flags = SCX_ENQ_HEAD; + ret = -1; + } + + scx_dispatch_enqueue(sch, donor_rq, dst_dsq, p, 0, 0, enq_flags); + + raw_spin_lock(&donor_dsq->lock); + return ret; +} + static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, struct cpumask *donee_mask, struct cpumask *resched_mask, u32 nr_donor_target, u32 nr_donee_target) @@ -6642,14 +6693,15 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, raw_spin_rq_lock_irq(donor_rq); raw_spin_lock(&donor_dsq->lock); + /* bound the walk to the tasks queued now, see the seq test below */ + cursor.priv = donor_dsq->seq; list_add(&cursor.node, &donor_dsq->list); resume: n = container_of(&cursor, struct task_struct, scx.dsq_list); n = nldsq_next_task(donor_dsq, n, false); while ((p = n)) { - struct scx_dispatch_q *donee_dsq; - int donee; + s32 donee, cpu; n = nldsq_next_task(donor_dsq, n, false); @@ -6659,6 +6711,13 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, if (cpumask_empty(donee_mask)) break; + /* + * A task queued after the walk began, including one the walk + * itself put back, is left for the next pass. + */ + if (u32_before(cursor.priv, p->scx.dsq_seq)) + continue; + /* * If an earlier pass placed @p on @donor_dsq from a different * CPU and the donee hasn't consumed it yet, @p is still on the @@ -6672,49 +6731,41 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, if (donee >= nr_cpu_ids) continue; - donee_dsq = scx_bypass_dsq(sch, donee); - /* - * $p's rq is not locked but $p's DSQ lock protects its - * scheduling properties making this test safe. + * The move drops @donor_dsq's lock. Park the cursor behind @p + * and resume from it afterwards. */ - if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) - continue; + list_move(&cursor.node, &p->scx.dsq_list.node); + cpu = bypass_lb_move(sch, donor_rq, donor_dsq, p, donee); + if (cpu >= 0) { + /* + * @cpu might have been idle and need to be woken up. + * No need to be clever. Reschedule every cpu that + * receives tasks. + */ + cpumask_set_cpu(cpu, resched_mask); - /* - * Moving $p from one non-local DSQ to another. The source rq - * and DSQ are already locked. Do an abbreviated dequeue and - * then perform enqueue without unlocking $donor_dsq. - * - * We don't want to drop and reacquire the lock on each - * iteration as @donor_dsq can be very long and potentially - * highly contended. Donee DSQs are less likely to be contended. - * The nested locking is safe as only this LB moves tasks - * between bypass DSQs. - */ - dispatch_dequeue_locked(p, donor_dsq); - scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, 0, 0, SCX_ENQ_NESTED); + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) >= nr_donee_target) + cpumask_clear_cpu(cpu, donee_mask); - /* - * $donee might have been idle and need to be woken up. No need - * to be clever. Kick every CPU that receives tasks. - */ - cpumask_set_cpu(donee, resched_mask); - - if (READ_ONCE(donee_dsq->nr) >= nr_donee_target) - cpumask_clear_cpu(donee, donee_mask); - - nr_balanced++; - if (!(nr_balanced % SCX_BYPASS_LB_BATCH) && n) { - list_move_tail(&cursor.node, &n->scx.dsq_list.node); - raw_spin_unlock(&donor_dsq->lock); - scx_rq_lock_drop(donor_rq); - raw_spin_rq_unlock_irq(donor_rq); - cpu_relax(); - raw_spin_rq_lock_irq(donor_rq); - raw_spin_lock(&donor_dsq->lock); - goto resume; + /* + * The rq lock is held across the walk, not cycled per + * move. @donor_dsq can be very long and contended, and + * the rq lock is what keeps its consumer and enqueuers + * away while the walk cycles the DSQ lock. The walk can + * still take a while, so let the cpu breathe every so + * often. + */ + if (!(++nr_balanced % SCX_BYPASS_LB_BATCH)) { + raw_spin_unlock(&donor_dsq->lock); + scx_rq_lock_drop(donor_rq); + raw_spin_rq_unlock_irq(donor_rq); + cpu_relax(); + raw_spin_rq_lock_irq(donor_rq); + raw_spin_lock(&donor_dsq->lock); + } } + goto resume; } list_del_init(&cursor.node); From a043733588a06e12e69918fbf080a6c45446e6a5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:47 -1000 Subject: [PATCH 29/34] sched_ext: Let the bypass load balancer move tasks homed on other cpus The following patch uses the bypass load balancer to drain a cpu's bypass DSQ when the host loses SCX_CAP_BASE there. The host no longer consumes that DSQ, so the drain has to move every task on it. The balancer skips the ones homed on another cpu. An earlier pass moved such a task in from its home cpu's DSQ, and the donor cpu has not consumed it yet. A task's DSQ membership may only change under the task's rq lock, which the balancer does not hold. Take such a task off the donor DSQ with unlink_dsq_and_switch_rq_lock(), the handshake a remote consumer uses. It drops the DSQ lock and switches to the task's rq lock. A dequeue that gets in between takes the task, and the balancer moves on. The insert then runs under the task's home rq lock, as it already does for a donor-homed task. A task whose donee is its own cpu can always run there. task_can_run_on_remote_rq() warns when asked about a task's own cpu, so the balancer skips the test for such a task. The walk holds the donor rq lock, which keeps the donor cpu's dispatch and its enqueuers away from the donor DSQ. A move of a task homed elsewhere switches to the task's rq lock, and both can reach the DSQ meanwhile. The parked cursor keeps the walk's place through a consume, and the sequence bound keeps a task enqueued meanwhile out of the pass. v2: The description says "the following patch", not "patches", uses the balancer to drain a DSQ. Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 66 +++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index cddfcc61c454..dfbfa6455629 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6624,7 +6624,7 @@ bool scx_hardlockup(int cpu) * @sch: host sched whose bypass DSQs are balanced * @donor_rq: rq of the cpu whose bypass DSQ gives up @p, locked * @donor_dsq: @sch's bypass DSQ on @donor_rq's cpu, locked - * @p: task on @donor_dsq, homed on @donor_rq + * @p: task on @donor_dsq, homed anywhere * @donee: cpu whose bypass DSQ receives @p * * Take @p off @donor_dsq and insert it into the bypass DSQ of @donee. The @@ -6632,28 +6632,45 @@ bool scx_hardlockup(int cpu) * can still refuse the move under @p's rq lock. @p then goes back to the head * of @donor_dsq. * - * @donor_rq and @donor_dsq are locked on entry and on return. @donor_dsq is - * unlocked across the insert, so the caller's walk must find its place in it - * again after the call. + * The insert runs under @p's rq lock. A task homed on @donor_rq is dequeued in + * place. unlink_dsq_and_switch_rq_lock() takes one homed elsewhere off and + * switches the rq lock from @donor_rq to its own. The switch can lose to a + * dequeue, and @p then belongs to the dequeuer. * - * Return the cpu that received @p, or -1 if @p went back onto @donor_dsq. + * @donor_rq and @donor_dsq are locked on entry and on return. @donor_dsq is + * unlocked in between, and @donor_rq too while the lock is switched, so the + * caller's walk must find its place in @donor_dsq again after the call. + * + * Return the cpu that received @p, or -1 if @p went back onto @donor_dsq or was + * lost to a dequeue. */ static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, struct scx_dispatch_q *donor_dsq, struct task_struct *p, s32 donee) { struct scx_dispatch_q *dst_dsq; u64 enq_flags; + struct rq *rq = task_rq(p); s32 ret; lockdep_assert_rq_held(donor_rq); lockdep_assert_held(&donor_dsq->lock); - scx_wait_dispatching(p); - dispatch_dequeue_locked(p, donor_dsq); - raw_spin_unlock(&donor_dsq->lock); + if (rq == donor_rq) { + scx_wait_dispatching(p); + dispatch_dequeue_locked(p, donor_dsq); + raw_spin_unlock(&donor_dsq->lock); + } else if (!unlink_dsq_and_switch_rq_lock(p, donor_dsq, donor_rq, rq)) { + /* lost to a dequeue, @p is the dequeuer's now */ + ret = -1; + goto out; + } - /* @p's affinity is stable under its rq lock */ - if (task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) { + /* + * @p's affinity is stable under its rq lock. A task whose donee is its + * own cpu can always run there, and task_can_run_on_remote_rq() warns + * when asked about a task's own cpu. + */ + if (donee == task_cpu(p) || task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) { dst_dsq = scx_bypass_dsq(sch, donee); enq_flags = 0; ret = donee; @@ -6664,8 +6681,10 @@ static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, ret = -1; } - scx_dispatch_enqueue(sch, donor_rq, dst_dsq, p, 0, 0, enq_flags); - + scx_dispatch_enqueue(sch, rq, dst_dsq, p, 0, 0, enq_flags); +out: + if (rq != donor_rq) + switch_rq_lock(rq, donor_rq); raw_spin_lock(&donor_dsq->lock); return ret; } @@ -6718,22 +6737,16 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, if (u32_before(cursor.priv, p->scx.dsq_seq)) continue; - /* - * If an earlier pass placed @p on @donor_dsq from a different - * CPU and the donee hasn't consumed it yet, @p is still on the - * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved - * without its rq locked. Skip. - */ - if (task_rq(p) != donor_rq) - continue; - donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); if (donee >= nr_cpu_ids) continue; /* - * The move drops @donor_dsq's lock. Park the cursor behind @p - * and resume from it afterwards. + * The move drops @donor_dsq's lock, and the rq lock too when + * @p is homed elsewhere. The donor cpu's dispatch and its + * enqueuers can reach @donor_dsq then. Park the cursor behind + * @p and resume from it afterwards. The seq test above keeps a + * task enqueued meanwhile out of the pass. */ list_move(&cursor.node, &p->scx.dsq_list.node); cpu = bypass_lb_move(sch, donor_rq, donor_dsq, p, donee); @@ -6752,9 +6765,10 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, * The rq lock is held across the walk, not cycled per * move. @donor_dsq can be very long and contended, and * the rq lock is what keeps its consumer and enqueuers - * away while the walk cycles the DSQ lock. The walk can - * still take a while, so let the cpu breathe every so - * often. + * away while the walk cycles the DSQ lock. The rare + * move of a task homed elsewhere does switch it away, + * see the cursor comment above. The walk can still take + * a while, so let the cpu breathe every so often. */ if (!(++nr_balanced % SCX_BYPASS_LB_BATCH)) { raw_spin_unlock(&donor_dsq->lock); From 5c23d1710606d3c14a0feafd8940df9f20bca7be Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:48 -1000 Subject: [PATCH 30/34] sched_ext: Fix queued bypassed tasks stalling when the host loses SCX_CAP_BASE A task already queued on a bypass DSQ stalls when the host loses SCX_CAP_BASE on that cpu, as the cpu no longer consumes the DSQ. Sending a bypassed task to rescue at enqueue time covers only a task enqueued after the loss, and the balancer no longer donates to a capless cpu. Have the bypass load balancer drain such a DSQ: a cpu where the host lacks BASE is a donor with a target of zero, and the donees take its tasks under their ordinary fill. A drained task no donee can take is reenqueued at its home: to rescue when the host lacks BASE there and rescue is enabled, and to the home's bypass DSQ otherwise. A balancing donor leaves a task that can't move to its donee where it is. Reenqueueing it at home would put a task homed on the donor cpu at the tail of its own DSQ, and a blocked task that proxy execution migrated there, which every remote cpu refuses, would go to the tail on every pass. Without rescue, a task homed on the drained cpu goes back onto the drained DSQ, once per pass, and waits for the host to regain BASE, the bypass to end, or the stall watchdog to eject the host. Its reenqueues are not counted against the reenqueue limit, as its scheduler is bypassing. A BASE loss kicks the balancer so that the drain doesn't wait for the next interval. With a zero interval the kick is the only pass, and a task that lands on the drained DSQ after it waits the same way. v2: The description drops an in-flight affinity change from the causes of a refused balancer move. v3: - The description says that a stuck task's reenqueues are not counted against the reenqueue limit (sashiko). - The description names the stall watchdog as the way out for a task stuck without rescue, and the tasks a zero interval leaves on a drained DSQ. - bypass_lb_reenq_home()'s kerneldoc and the description give the placement with rescue disabled and for a task that must run at home, and the other two comments on that placement point to the kerneldoc. v4: - Skip a task the donee would refuse on a balancing donor instead of reenqueueing it at home (Andrea). - bypass_lb_reenq_home()'s kerneldoc is corrected on a migration-disabled task homed on a donee and on the local DSQ admission. - The description is condensed. Fixes: 147d1885f390 ("sched_ext: Add the SCX_CAP_ENQ_IMMED cap") Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 163 ++++++++++++++++++++++++++---------- kernel/sched/ext/internal.h | 1 + kernel/sched/ext/sub.c | 8 ++ 3 files changed, 129 insertions(+), 43 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index dfbfa6455629..57cb6886efb5 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -6619,36 +6619,76 @@ bool scx_hardlockup(int cpu) return true; } +/** + * bypass_lb_reenq_home - Reenqueue a task at its home from the balancer + * @rq: @p's home rq, locked + * @p: task off any DSQ, taken from the bypass DSQ of @dsq_cpu + * @dsq_cpu: cpu whose bypass DSQ gave up @p + * + * @p came off the bypass DSQ of @dsq_cpu with no donee able to take it. A drain + * sends a migration-disabled @p here unless the donee picked for it is its + * home, as every other cpu refuses it. The enqueue sends @p to rescue when the + * host lacks %SCX_CAP_BASE at home and rescue is enabled, and to the home's + * bypass DSQ otherwise. When @p must run at home, migration-disabled or with a + * migration pending, and the host lacks BASE there, it is admitted to the + * home's local DSQ in either case, see bypass_enq_target_dsq(). No DSQ lock may + * be held, as the enqueue takes one. + */ +static void bypass_lb_reenq_home(struct rq *rq, struct task_struct *p, s32 dsq_cpu) +{ + lockdep_assert_rq_held(rq); + + WARN_ON_ONCE(p->scx.flags & SCX_TASK_REENQ_REASON_MASK); + p->scx.flags |= SCX_TASK_REENQ_CAP; + p->scx.reenq_reason_caps = SCX_CAP_BASE; + p->scx.reenq_reason_cid = __scx_cpu_to_cid(dsq_cpu); + scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); +} + +/** + * bypass_lb_can_move - Test whether @p can move to the bypass DSQ of @donee + * @sch: host sched whose bypass DSQs are balanced + * @p: task on a bypass DSQ of @sch, homed anywhere + * @donee: cpu whose bypass DSQ would receive @p, -1 for none + * + * @p's rq lock pins @p's affinity. Without it the answer is a hint. A task + * whose donee is its own cpu can always run there, and + * task_can_run_on_remote_rq() warns when asked about a task's own cpu. + */ +static bool bypass_lb_can_move(struct scx_sched *sch, struct task_struct *p, s32 donee) +{ + return donee >= 0 && (donee == task_cpu(p) || + task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)); +} + /** * bypass_lb_move - Move one task off a bypass DSQ for the balancer * @sch: host sched whose bypass DSQs are balanced * @donor_rq: rq of the cpu whose bypass DSQ gives up @p, locked * @donor_dsq: @sch's bypass DSQ on @donor_rq's cpu, locked * @p: task on @donor_dsq, homed anywhere - * @donee: cpu whose bypass DSQ receives @p + * @donee: cpu whose bypass DSQ receives @p, -1 for none * * Take @p off @donor_dsq and insert it into the bypass DSQ of @donee. The - * caller picks @donee from @p's affinity alone and task_can_run_on_remote_rq() - * can still refuse the move under @p's rq lock. @p then goes back to the head - * of @donor_dsq. + * caller tests bypass_lb_can_move() without @p's rq lock for a task homed + * elsewhere, and not at all for a drain, so the test is repeated here under + * @p's rq lock. A task that fails it is not put back, as the host may no longer + * consume @donor_dsq. bypass_lb_reenq_home() reenqueues it at its home instead. * - * The insert runs under @p's rq lock. A task homed on @donor_rq is dequeued in - * place. unlink_dsq_and_switch_rq_lock() takes one homed elsewhere off and - * switches the rq lock from @donor_rq to its own. The switch can lose to a - * dequeue, and @p then belongs to the dequeuer. + * The insert and the reenqueue run under @p's rq lock. A task homed on + * @donor_rq is dequeued in place. unlink_dsq_and_switch_rq_lock() takes one + * homed elsewhere off and switches the rq lock from @donor_rq to its own. The + * switch can lose to a dequeue, and @p then belongs to the dequeuer. * * @donor_rq and @donor_dsq are locked on entry and on return. @donor_dsq is * unlocked in between, and @donor_rq too while the lock is switched, so the * caller's walk must find its place in @donor_dsq again after the call. * - * Return the cpu that received @p, or -1 if @p went back onto @donor_dsq or was - * lost to a dequeue. + * Return the cpu that received @p, or -1 if @p was lost to a dequeue. */ static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, struct scx_dispatch_q *donor_dsq, struct task_struct *p, s32 donee) { - struct scx_dispatch_q *dst_dsq; - u64 enq_flags; struct rq *rq = task_rq(p); s32 ret; @@ -6665,23 +6705,17 @@ static s32 bypass_lb_move(struct scx_sched *sch, struct rq *donor_rq, goto out; } - /* - * @p's affinity is stable under its rq lock. A task whose donee is its - * own cpu can always run there, and task_can_run_on_remote_rq() warns - * when asked about a task's own cpu. - */ - if (donee == task_cpu(p) || task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false)) { - dst_dsq = scx_bypass_dsq(sch, donee); - enq_flags = 0; + if (bypass_lb_can_move(sch, p, donee)) { + scx_dispatch_enqueue(sch, rq, scx_bypass_dsq(sch, donee), p, 0, 0, 0); ret = donee; } else { - /* back where it was, at the head so that the walk skips it */ - dst_dsq = donor_dsq; - enq_flags = SCX_ENQ_HEAD; - ret = -1; + /* + * No donee, or @p can't run on it. Reenqueue @p at home + * instead, see bypass_lb_reenq_home(). + */ + bypass_lb_reenq_home(rq, p, cpu_of(donor_rq)); + ret = cpu_of(rq); } - - scx_dispatch_enqueue(sch, rq, dst_dsq, p, 0, 0, enq_flags); out: if (rq != donor_rq) switch_rq_lock(rq, donor_rq); @@ -6704,10 +6738,11 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, * All we want to guarantee is reasonable forward progress. No reason to * fine tune. Assuming every task on @donor_dsq runs their full slice, * consider offloading iff the total queued duration is over the - * threshold. + * threshold. A zero target is a drain and empties the DSQ regardless. */ min_delta_us = READ_ONCE(scx_bypass_lb_intv_us) / SCX_BYPASS_LB_MIN_DELTA_DIV; - if (delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) + if (nr_donor_target && + delta < DIV_ROUND_UP(min_delta_us, READ_ONCE(scx_slice_bypass_us))) return 0; raw_spin_rq_lock_irq(donor_rq); @@ -6727,18 +6762,29 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor, if (donor_dsq->nr <= nr_donor_target) break; - if (cpumask_empty(donee_mask)) + /* + * Without a donee, a balancing donor has nothing to do. A drain + * still sends its tasks home. + */ + if (cpumask_empty(donee_mask) && nr_donor_target) break; /* * A task queued after the walk began, including one the walk - * itself put back, is left for the next pass. + * itself reenqueued, is left for the next pass. */ if (u32_before(cursor.priv, p->scx.dsq_seq)) continue; + /* + * A balancing donor leaves a task that can't move where it is. + * A home reenqueue would push one homed on the donor to the + * back of @donor_dsq on every pass. A drain sends it home. + */ donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); if (donee >= nr_cpu_ids) + donee = -1; + if (nr_donor_target && !bypass_lb_can_move(sch, p, donee)) continue; /* @@ -6795,6 +6841,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) const struct cpumask *node_mask = cpumask_of_node(node); struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask; struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask; + u32 intv_us = READ_ONCE(scx_bypass_lb_intv_us); u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0; u32 nr_target, nr_donor_target; u32 before_min = U32_MAX, before_max = 0; @@ -6814,7 +6861,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) before_max = max(nr, before_max); } - if (!nr_cpus) + if (!nr_tasks) return; /* @@ -6822,7 +6869,7 @@ static void bypass_lb_node(struct scx_sched *sch, int node) * balancing to fill donee CPUs up to $nr_target. Once targets are * calculated, find the donee CPUs. */ - nr_target = DIV_ROUND_UP(nr_tasks, nr_cpus); + nr_target = nr_cpus ? DIV_ROUND_UP(nr_tasks, nr_cpus) : 0; nr_donor_target = DIV_ROUND_UP(nr_target * SCX_BYPASS_LB_DONOR_PCT, 100); cpumask_clear(donee_mask); @@ -6833,18 +6880,28 @@ static void bypass_lb_node(struct scx_sched *sch, int node) cpumask_set_cpu(cpu, donee_mask); } - /* iterate !donee CPUs and see if they should be offloaded */ + /* + * Offload the !donee cpus. A cpu where the host lost %SCX_CAP_BASE no + * longer consumes its DSQ. Drain it as a donor with a target of zero. A + * zero interval disables the balancing, not the drain. + */ cpumask_clear(resched_mask); for_each_cpu_and(cpu, cpu_online_mask, node_mask) { - if (cpumask_empty(donee_mask)) - break; - if (cpumask_test_cpu(cpu, donee_mask)) + u32 target = nr_donor_target; + + if (unlikely(scx_missing_caps(sch, cpu, SCX_CAP_BASE))) { + /* the mask may predate the loss, keep this cpu out */ + cpumask_clear_cpu(cpu, donee_mask); + target = 0; + } else if (!intv_us || cpumask_empty(donee_mask) || + cpumask_test_cpu(cpu, donee_mask)) { continue; - if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= nr_donor_target) + } + if (READ_ONCE(scx_bypass_dsq(sch, cpu)->nr) <= target) continue; nr_balanced += bypass_lb_cpu(sch, cpu, donee_mask, resched_mask, - nr_donor_target, nr_target); + target, nr_target); } for_each_cpu(cpu, resched_mask) @@ -6883,9 +6940,28 @@ static void scx_bypass_lb_timerfn(struct timer_list *timer) for_each_node_with_cpus(node) bypass_lb_node(sch, node); + /* + * A kick during this run can leave the timer pending at now, see + * scx_bypass_lb_kick(). timer_reduce() only brings the expiry forward, + * so that kick is kept. + */ intv_us = READ_ONCE(scx_bypass_lb_intv_us); if (intv_us) - mod_timer(timer, jiffies + usecs_to_jiffies(intv_us)); + timer_reduce(timer, jiffies + usecs_to_jiffies(intv_us)); +} + +/** + * scx_bypass_lb_kick - Run the bypass load balancer now + * @sch: sched that lost %SCX_CAP_BASE on a cpu + * + * @sch no longer consumes its bypass DSQ on that cpu. Have the balancer drain + * it without waiting for the next interval, see bypass_lb_node(). A loss during + * a run leaves the timer pending at now, and the run that follows sees it. + */ +void scx_bypass_lb_kick(struct scx_sched *sch) +{ + if (scx_bypass_dsp_enabled(sch)) + mod_timer(&sch->bypass_lb_timer, jiffies); } static bool inc_bypass_depth(struct scx_sched *sch) @@ -6954,11 +7030,12 @@ static void enable_bypass_dsp(struct scx_sched *sch) /* * The LB timer will stop running if bypass dispatch is disabled. Start - * after enabling bypass dispatch. + * after enabling bypass dispatch. A kick from a BASE loss can have set + * the timer to expire now in the meantime, see scx_bypass_lb_kick(). + * timer_reduce() keeps that expiry. */ - if (intv_us && !timer_pending(&host->bypass_lb_timer)) - mod_timer(&host->bypass_lb_timer, - jiffies + usecs_to_jiffies(intv_us)); + if (intv_us) + timer_reduce(&host->bypass_lb_timer, jiffies + usecs_to_jiffies(intv_us)); } /* may be called without holding scx_bypass_lock */ diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index ecd677c4b561..ff0a3d508533 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -2332,6 +2332,7 @@ void scx_cgroup_unlock(void); s32 scx_alloc_kern_arena_objs(struct scx_sched *sch); void scx_disable_bypass_dsp(struct scx_sched *sch); void scx_bypass(struct scx_sched *sch, bool bypass); +void scx_bypass_lb_kick(struct scx_sched *sch); s32 scx_link_sched(struct scx_sched *sch); void scx_unlink_sched(struct scx_sched *sch); void scx_disable_dump(struct scx_sched *sch); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 01215384eaa5..ecab82ca5e82 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1243,6 +1243,14 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) gained = ecaps & ~old; lost_all |= lost; + /* + * The bypass DSQ on this cpu is no longer consumed, so drain + * it. A bypassing sched's own bypass DSQs are empty, as its + * tasks sit on the host's. + */ + if ((lost & SCX_CAP_BASE) && !scx_bypassing(pcpu->sch, cpu)) + scx_bypass_lb_kick(pcpu->sch); + /* * Tell the sched and its parent that the sched's effective caps * on this cid changed. The invocations are equivalent to the From 5173b2965834ad71d8c9035048354dce7c391125 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 26 Oct 2026 14:17:49 -1000 Subject: [PATCH 31/34] sched_ext: Fix a lone IMMED task waiting behind a kept prev While dispatch is in progress, rq_is_open() returns true and dsq_inc_nr() schedules no reenqueue for an IMMED task inserted on an empty local DSQ. The check at the end of dispatch_one() schedules the reenqueue only when more than one task is queued, so a dispatch that keeps prev leaves a lone IMMED task behind: 1. a parent's ops.dispatch() sub-dispatches into a child, which consumes a task with SCX_ENQ_IMMED onto the empty local DSQ 2. the parent extends prev's slice and dispatch keeps prev 3. the local DSQ holds one task, so no reenqueue is scheduled The IMMED task waits on the local DSQ behind prev. Schedule the reenqueue when dispatch keeps prev and an IMMED task is queued, as even a lone IMMED task can't run right away behind prev. v2: The description and dispatch_one()'s comment are reworded. Fixes: 98d709cba319 ("sched_ext: Implement SCX_ENQ_IMMED") Reported-by: Sashiko Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 57cb6886efb5..3f1ebe2edbc7 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -3500,8 +3500,12 @@ static enum scx_dsp_verdict dispatch_one(struct rq *rq, struct task_struct *prev * * - A non-IMMED HEAD task can get queued in front of an IMMED task * between the IMMED queueing and the subsequent scheduling event. + * + * - When dispatch keeps @prev, even a lone IMMED task queued during the + * dispatch can't run right away. */ - if (unlikely(rq->scx.local_dsq.nr > 1 && rq->scx.nr_immed)) + if (unlikely((rq->scx.local_dsq.nr > 1 || verdict == SCX_DSP_PREV) && + rq->scx.nr_immed)) scx_schedule_reenq_local(rq, 0); rq->scx.flags &= ~SCX_RQ_IN_DISPATCH; From a298fd5f4ee8f7e80c4dc74d9c501c22a037c6b6 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 27 Oct 2026 04:08:47 -1000 Subject: [PATCH 32/34] sched_ext: Fix use-after-free of a NONE rescuee leaving its sched while running Since an earlier patch sends a bypassed task to rescue when its host lacks SCX_CAP_BASE, a task scx_fail_parent() punted to its bypassing parent at NONE can be a rescuee. When a cgroup migration into a root-owned cgroup re-homes it while running, nothing resets it: the DEQUEUE_SAVE dequeue and put_prev_task_scx() keep a running task's slice, and __scx_disable_and_exit_task() skips a NONE task. rq->scx.rescue.curr keeps pointing at the task, now on the root. An RT task, for example, can keep the task from serving its grant until the last sub is freed. From then on scx_rescuee() returns NULL and the rescue never ends. scx_rescue_timerfn() keeps accessing the task after it is freed. Move the reset out of scx_disable_task() into scx_reset_task_fields() and call it on every path that takes a task out of its sched's control, the exit of a NONE task included. Reported-by: Sashiko Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/ext.c | 66 +++++++++++++++++++++++++++--------------- kernel/sched/ext/sub.c | 4 +-- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 3f1ebe2edbc7..c173820acfce 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -2511,8 +2511,9 @@ static void ops_dequeue(struct rq *rq, struct task_struct *p, u64 deq_flags) /* * Don't reset runnable_at. If @p is waiting, the wait continues across * temporary dequeues, e.g. for a migration or an attribute change. - * scx_disable_task() ends it if @p leaves its scheduler. scx_bypass() - * restarts it when its requeue can change the sched running @p. + * scx_reset_task_fields() ends it if @p leaves its sched's control. + * scx_bypass() restarts it when its requeue can change the sched + * running @p. */ clr_task_runnable(p, false); @@ -4657,6 +4658,34 @@ void scx_enable_task(struct scx_sched *sch, struct task_struct *p) scx_set_task_state(p, SCX_TASK_ENABLED); } +/** + * scx_reset_task_fields - Reset what @p carries from its sched's control + * @rq: @p's rq, locked + * @p: task leaving its sched's control + * + * The vtime, the slice and what rides on it, the reenqueue count and the + * watchdog wait belong to the sched while it controls @p. The control ends when + * @p is taken off the sched or leaves the ext class while staying on the sched. + * Reset them so that @p starts clean wherever it goes. + */ +static void scx_reset_task_fields(struct rq *rq, struct task_struct *p) +{ + lockdep_assert_rq_held(rq); + + p->scx.dsq_vtime = 0; + scx_task_slice_ended(rq, p); + scx_set_task_slice(p, 0); + p->scx.reenq_cnt = 0; + + /* + * If @p is runnable but not running, its wait started at runnable_at, + * which ops_dequeue() didn't reset. Leaving the sched's control ends + * the wait. Reset runnable_at the next time @p is enqueued on SCX, so + * that the watchdog doesn't count the time @p spends elsewhere. + */ + p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; +} + static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) { struct rq *rq = task_rq(p); @@ -4670,23 +4699,6 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p) SCX_CALL_OP_TASK(sch, disable, rq, p); scx_set_task_state(p, SCX_TASK_READY); - /* - * Reset the SCX-managed fields when @p leaves the BPF scheduler's - * control, after ops.disable() has observed their final values. - */ - p->scx.dsq_vtime = 0; - scx_task_slice_ended(rq, p); - scx_set_task_slice(p, 0); - p->scx.reenq_cnt = 0; - - /* - * If @p is runnable but not running, its wait started at runnable_at, - * which ops_dequeue() didn't reset. Leaving @sch ends the wait. Reset - * runnable_at the next time @p is enqueued on SCX, so that the watchdog - * doesn't count the time @p spends elsewhere. - */ - p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT; - /* * Verify the task is not in BPF scheduler's custody. If flag * transitions are consistent, the flag should always be clear @@ -4706,6 +4718,11 @@ void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) switch (scx_get_task_state(p)) { case SCX_TASK_NONE: + /* + * No op to call, but @p may still carry @sch's state, e.g. a + * task punted by scx_fail_parent() and being rescued. + */ + scx_reset_task_fields(task_rq(p), p); return; case SCX_TASK_INIT: args.cancelled = true; @@ -4714,6 +4731,8 @@ void __scx_disable_and_exit_task(struct scx_sched *sch, struct task_struct *p) break; case SCX_TASK_ENABLED: scx_disable_task(sch, p); + /* after ops.disable() has seen the final values */ + scx_reset_task_fields(task_rq(p), p); break; default: WARN_ON_ONCE(true); @@ -5022,13 +5041,12 @@ static void switched_from_scx(struct rq *rq, struct task_struct *p) * scx_disable_task() would WARN on the non-%ENABLED state and trigger a * NONE -> READY validation failure. * - * @p can still be a rescuee under the parent. End its slice as - * scx_disable_task() does for an enabled task. + * Either way, reset @p after ops.disable() has seen the final values. A + * %NONE @p can still be a rescuee under the parent. */ - if (scx_get_task_state(p) == SCX_TASK_NONE) - scx_task_slice_ended(rq, p); - else + if (scx_get_task_state(p) != SCX_TASK_NONE) scx_disable_task(scx_task_sched(p), p); + scx_reset_task_fields(rq, p); } static void switched_to_scx(struct rq *rq, struct task_struct *p) {} diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index ecab82ca5e82..e0aed3ce9735 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -1666,8 +1666,8 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) * Used when @to must take over @p but failed to initialize it. Bypass keeps * scheduling decisions away from @to but @p can still trigger its task ops, * which may confuse the BPF side. @to is dying anyway. The exit paths call no - * op on a %NONE task. __scx_disable_and_exit_task() skips it and - * switched_from_scx() only ends its slice. + * op on a %NONE task. __scx_disable_and_exit_task() and switched_from_scx() + * only reset its fields, see scx_reset_task_fields(). */ static void scx_punt_task(struct scx_sched *to, struct task_struct *p) { From b7723dedac0e92e5a74e89cb800a0083c2ab9ee2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 27 Oct 2026 04:08:48 -1000 Subject: [PATCH 33/34] sched_ext: Fix the rescue overload pick ejecting the wrong sub under bypass An earlier patch sends a bypassed task to rescue when its host lacks SCX_CAP_BASE on the cpu. The task's rescue time is charged to its owner, the bypassing sub, which is usually exiting. The overload pick skips exiting subs and ejects an unrelated sub or nobody. The host runs the task and lacks BASE, but carries none of the charge. Charge the sched running the rescuee, the host while the owner bypasses. SCX_EV_SUB_RESCUE already counts on that sched and the stall watchdog blames it. Reported-by: Sashiko Signed-off-by: Tejun Heo Reviewed-by: Andrea Righi --- kernel/sched/ext/sub.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index e0aed3ce9735..068ca57d1bb7 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -479,9 +479,13 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec) rq->scx.rescue.budget -= delta_exec; - /* per-cpu usage average feeds the overload victim pick */ - pcpu = per_cpu_ptr(scx_task_sched(rq->scx.rescue.curr)->pcpu, - cpu_of(rq)); + /* + * The per-cpu usage average feeds the overload victim pick. Charge the + * sched running the rescuee, which is the host while the rescuee's + * owner is bypassing. + */ + pcpu = per_cpu_ptr(scx_exec_sched(scx_task_sched(rq->scx.rescue.curr), + cpu_of(rq))->pcpu, cpu_of(rq)); pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec; if (!scx_rescue_slice_remaining(rq)) @@ -665,7 +669,8 @@ static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p) * If the oldest waiter on @rq's rescue DSQ has been queued for too long, rescue * demand on this cpu persistently exceeds the configured bandwidth. Eject the * sub with the highest recent rescue consumption instead of letting the - * scheduler stall path blame the waiter's owner, who may just be crowded out. + * scheduler stall path blame the sched running the waiter, which may just be + * crowded out. */ static void scx_rescue_check_overload(struct rq *rq) { From af4ed5682e62556beb399b76f2f1274ae1aad9e2 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 27 Oct 2026 08:03:53 -1000 Subject: [PATCH 34/34] sched_ext: Reject scx_bpf_sub_grant/revoke() in tracing programs A tracing program can call scx_bpf_sub_grant() and scx_bpf_sub_revoke() through scx_kfunc_ids_any, which is registered on the tracing kfunc hook. Before returning, both invoke ops.sub_caps_updated() of the sub-schedulers whose caps changed, so those ops run in the tracing program's context. No SCX op expects to run from an arbitrary tracing context. With an rq lock held, this leads to the same deadlock that b07a7450f6ef ("sched_ext: Reject rq-lock-taking kfuncs in tracing programs") fixed for direct calls: 1. A tp_btf/sched_switch program fires from __schedule() with the rq lock held. scx_locked_rq() only tracks rq locks held across SCX callbacks, so it returns NULL there. 2. The program calls scx_bpf_sub_grant() on a child, which invokes the child's ops.sub_caps_updated() before returning. 3. That op calls scx_bpf_cidperf_set() on this cpu, which sees no locked rq and takes the rq lock the cpu already holds. Add both kfuncs to scx_kfunc_ids_no_tracing and rewrite the set's comment to cover both reasons for membership: taking an rq lock and running SCX ops. Fixes: 5f2a9a4c2e6c ("sched_ext: Add coalescing sub_caps_updated() notifier for sub-schedulers") Signed-off-by: Tejun Heo --- kernel/sched/ext/ext.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index c173820acfce..1b098728676f 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -12126,19 +12126,28 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = { }; /* - * Kfuncs which may take the rq lock on their unlocked path. + * Kfuncs that tracing programs can't call. A tracing program can run in any + * kernel context, including with an rq lock held. scx_locked_rq() only tracks + * rq locks held across SCX callbacks and returns NULL there. * - * A tracing program can execute in any kernel context, including ones which - * already hold the rq lock (e.g. sched_switch). scx_locked_rq() only tracks - * rq locks taken for SCX callbacks, so from such a context it returns NULL - * and these kfuncs would acquire the already held rq lock again and deadlock - * the CPU. They cannot be made safe for tracing programs, so keep them off - * the tracing hook. Members must also stay in scx_kfunc_ids_any so that - * struct_ops and syscall programs keep their access. + * scx_bpf_cpuperf_set() and scx_bpf_cidperf_set() take the target cpu's rq lock + * when scx_locked_rq() returns NULL. Taking it while the context already holds + * an rq lock, the same one or another cpu's, can deadlock. + * + * Before returning, scx_bpf_sub_grant() and scx_bpf_sub_revoke() invoke + * ops.sub_caps_updated() of the sub-schedulers whose caps changed, so those ops + * run in the tracing program's context. No SCX op expects to run from an + * arbitrary tracing context. One that calls scx_bpf_cidperf_set() hits the + * deadlock above. + * + * Members must also stay in scx_kfunc_ids_any so that struct_ops and syscall + * programs keep their access. */ BTF_KFUNCS_START(scx_kfunc_ids_no_tracing) BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_grant, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, scx_bpf_sub_revoke, KF_IMPLICIT_ARGS) BTF_KFUNCS_END(scx_kfunc_ids_no_tracing) /*