diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 255607739fb1..f37a4c9d4121 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1152,6 +1152,36 @@ static void schedule_deferred_locked(struct rq *rq) schedule_deferred(rq); } +static void resched_irq_workfn(struct irq_work *irq_work) +{ + struct rq *rq = container_of(irq_work, struct rq, scx.resched_irq_work); + + guard(rq_lock)(rq); + /* a higher class task would only be picked again, see kick_one_cpu() */ + if (!sched_class_above(rq->donor->sched_class, &ext_sched_class)) + resched_curr(rq); + scx_rq_lock_drop(rq); +} + +/** + * scx_resched_curr_deferred - Reschedule @rq's cpu from hard irq work on it + * @rq: rq to reschedule + * + * A kick, like a direct resched, only guarantees that a scheduling event on the + * cpu completes after it, and that can be the one in progress. Work that a pick + * processes only at the start of its dispatch, such as the ecaps syncs, can be + * queued after that point and miss the pick. A pick on @rq's cpu runs with IRQs + * off, so the irq work runs after it, and the scheduling event its resched + * triggers starts after this call. A sibling's core scheduling pick for @rq + * isn't covered. A cpu running a task of a higher class isn't rescheduled, as + * its pick would return that task again. @rq's cpu must stay online until the + * irq work is queued. + */ +void scx_resched_curr_deferred(struct rq *rq) +{ + irq_work_queue_on(&rq->scx.resched_irq_work, cpu_of(rq)); +} + #ifdef CONFIG_NO_HZ_FULL static void scx_proxy_tick_bal_cb(struct rq *rq) { @@ -6791,6 +6821,9 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * * - ops.dispatch() is ignored. * + * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called. + * The bypassed tasks are on the bypass DSQs, out of their reach. + * * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice * can't be trusted. Whenever a tick triggers, the running task is rotated to * the tail of the queue. @@ -6870,10 +6903,8 @@ void scx_bypass(struct scx_sched *sch, bool bypass) pcpu->flags |= SCX_SCHED_PCPU_BYPASSING; } else { pcpu->flags &= ~SCX_SCHED_PCPU_BYPASSING; - if (was_bypassing) { + if (was_bypassing) unbypass_renotify_idle(rq, pos, pcpu); - scx_unbypass_replay_ecaps(rq, pos); - } } } @@ -9510,6 +9541,7 @@ void __init init_sched_ext_class(void) INIT_LIST_HEAD(&rq->scx.deferred_reenq_users); rq->scx.deferred_irq_work = IRQ_WORK_INIT_HARD(deferred_irq_workfn); rq->scx.kick_cpus_irq_work = IRQ_WORK_INIT_HARD(kick_cpus_irq_workfn); + rq->scx.resched_irq_work = IRQ_WORK_INIT_HARD(resched_irq_workfn); if (cpu_online(cpu)) cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE; diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index 29e69a83d876..a3c63522c38a 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -878,9 +878,14 @@ struct sched_ext_ops { * @args: argument container, see the struct definition * * The sub-scheduler holds no caps by this point and can no longer - * affect any cid. Its caps went away without a - * sub_child_ecaps_updated() report, so whatever was delegated to it, - * e.g. cpuperf targets, is the parent's to restore here. + * affect any cid. Every cap reported through + * ops.sub_child_ecaps_updated() was also reported lost there before the + * sub-scheduler's tasks came back to this sched, except on a cpu that + * went offline. Nothing is reported for a sub-scheduler whose enable + * failed before its tasks were handed over. If this sched is being + * disabled too, it can miss these reports. On a cpu whose loss wasn't + * reported, whatever was delegated, e.g. cpuperf targets, is the + * parent's to restore here. */ void (*sub_detach)(struct scx_sub_detach_args *args); @@ -914,6 +919,14 @@ struct sched_ext_ops { * change is in effect on the cpu. Runs in dispatch context with rq lock * held, and can perform all operations allowed in ops.dispatch() * including inserting/moving tasks. + * + * Deliveries start after this sched's tasks are handed to it and before + * it makes its first scheduling decision. The caps granted from + * ops.sub_attach() and any change since are reported then. A cpu that + * isn't online at that point reports them once it comes online. + * Deliveries continue while this sched bypasses. Its tasks are then on + * the bypass DSQs, where this op can't insert or move them. Deliveries + * stop once this sched is being disabled. */ void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after); @@ -940,11 +953,16 @@ struct sched_ext_ops { * @before: the child's effective caps as of the last delivery * @after: the child's effective caps now * - * Invoked right after the child's sub_ecaps_updated(), once a grant or - * revoke is in effect on the cpu. From here on the parent can act on a + * Invoked right after the child's ops.sub_ecaps_updated(), once a + * change is in effect on the cpu. From here on the parent can act on a * cap the child lost, e.g. reset the cpuperf target or schedule on the - * cid. A detaching child's caps go away without a report, see - * sub_detach(). A cpu going offline drops caps without a report. + * cid. A disabling child gets no ops.sub_ecaps_updated(), so only this + * op is invoked for its loss. A cpu going offline drops caps without a + * report. + * + * Deliveries start as for the child's ops.sub_ecaps_updated(), continue + * while either sched bypasses and stop once this sched is being + * disabled. * * Runs in dispatch context with rq lock held, and can perform all * operations allowed in ops.dispatch() including inserting/moving @@ -1414,6 +1432,10 @@ struct scx_sched; enum scx_sched_pcpu_flags { SCX_SCHED_PCPU_BYPASSING = 1LLU << 0, + /* ecaps deliveries are enabled, see ecaps_enable() */ + SCX_SCHED_PCPU_ECAPS_ENABLED = 1LLU << 1, + /* a report is pending, see ecaps_report() */ + SCX_SCHED_PCPU_ECAPS_REPORT = 1LLU << 2, }; /* dispatch buf */ @@ -1474,6 +1496,16 @@ struct scx_sched_pcpu { struct llist_node ecaps_to_sync_node; /* owed a forced update_idle() re-notify on this cpu */ bool idle_renotify; + /* + * The sched was granted a cap on this cpu at some point. ecaps_report() + * reschedules only these cpus, as rescheduling every cpu on each + * sub-sched enable and disable would disrupt a large machine split many + * ways. Exactly which cpus owe a delivery is hard to track, as + * deliveries are asynchronous and reading caps[] and ecaps doesn't + * tell. A cpu granted a cap at some point is a simple superset. Written + * under pshard->lock. + */ + bool had_caps; /* * ecaps as of the last sub_ecaps_updated() and * sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps(). @@ -1751,6 +1783,11 @@ struct scx_sched { bool linked; /* on ->children, see scx_link_sched() */ bool sub_attached; + /* ecaps_enable() has run, under scx_enable_mutex */ + bool ecaps_enabled; + + /* marked cpus whose sync is pending, see ecaps_report() */ + atomic_t nr_ecaps_reports; #endif /* CONFIG_EXT_SUB_SCHED */ /* @@ -2260,6 +2297,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq); s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch); __printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...); void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); +void scx_resched_curr_deferred(struct rq *rq); u64 __scx_bpf_now(struct rq *rq); void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, u64 reenq_flags, struct rq *locked_rq); diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 1923bdc33d6b..a3f7c0be98ac 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -42,6 +42,17 @@ static s64 scx_rescue_sat_delta_ns; static unsigned long scx_rescue_decay_halflife; static unsigned long scx_rescue_overload_after; +/* + * ecaps_report() waits here for its marked syncs. A cpu going down keeps + * picking tasks after hotplug parks its irq_work thread. Its picks can process + * a marked sync, e.g. a core scheduling sibling's. A regular irq_work queued + * there is left behind on PREEMPT_RT, so the wakeup is hard irq work. A hard + * irq can't take a wait queue's lock, which sleeps on PREEMPT_RT. + * scx_enable_mutex serializes the reports, so the single waiter sleeps on an + * rcuwait. + */ +static struct rcuwait ecaps_report_wait = __RCUWAIT_INITIALIZER(ecaps_report_wait); + /** * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk * @pos: current position @@ -964,7 +975,7 @@ static void caps_updated_deliver(struct list_head *to_deliver) * so !has_op means the op is absent or the sched isn't * live yet - e.g. caps grant from ops.sub_attach(). * Either way don't consume - leave for - * scx_sub_seed_caps() to deliver once live. + * scx_sub_seed_caps() to deliver once has_op is set. */ scoped_guard (raw_spinlock, &cu->lock) { if (cu->caps && SCX_HAS_OP(sch, sub_caps_updated) && @@ -992,8 +1003,8 @@ static void caps_updated_deliver(struct list_head *to_deliver) /* * Deliver caps owed to @sch that couldn't be delivered earlier (e.g. a grant - * taken during its sub_attach(), before has_op was set). Called once @sch is - * enabled. + * taken during its sub_attach(), before has_op was set). Called once has_op is + * set. */ static void scx_sub_seed_caps(struct scx_sched *sch) { @@ -1030,8 +1041,12 @@ static u64 calc_effective_caps(struct scx_pshard *ps, s32 cid) * @sch: sched to update * @cid: cid to update * - * Queue an ecaps update for @sch's @cid and kick the cpu so that it syncs in - * dispatch_one(). + * Queue an ecaps update for @sch's @cid and reschedule the cpu so that it syncs + * in dispatch_one(). The cpu isn't rescheduled while it is offline or the root + * bypasses. Its next scheduling event processes the sync then, and a cpu coming + * back syncs again in the reseed. A grant or revoke can run in the PM syscore + * phase, where irq work can malfunction, see scx_kick_cpu(), and the root + * bypasses throughout PM. ecaps_report() reschedules the cpu itself. */ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) { @@ -1048,8 +1063,15 @@ static void queue_sync_ecaps(struct scx_sched *sch, s32 cid) /* @cid's pshard->lock excludes concurrent queueing attempts */ if (llist_on_list(&pcpu->ecaps_to_sync_node)) return; - if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync)) - scx_kick_cpu(sch->ancestors[0], cpu, 0); + + /* + * The irq work needs an online cpu. An online cpu stays online here, as + * @cid's pshard->lock keeps preemption off, which holds off the + * stop_machine() that takes a cpu offline. + */ + if (llist_add(&pcpu->ecaps_to_sync_node, &cpu_rq(cpu)->scx.ecaps_to_sync) && + cpu_online(cpu) && !scx_bypassing(sch->ancestors[0], smp_processor_id())) + scx_resched_curr_deferred(cpu_rq(cpu)); } /* discard @rq's queued ecaps syncs */ @@ -1069,14 +1091,31 @@ static struct scx_pshard *cid_pshard(struct scx_sched *sch, s32 cid) return sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; } +static void ecaps_report_irq_workfn(struct irq_work *irq_work) +{ + rcuwait_wake_up(&ecaps_report_wait); +} + +static struct irq_work ecaps_report_irq_work = IRQ_WORK_INIT_HARD(ecaps_report_irq_workfn); + +/* + * Complete @pcpu's report mark. The waiter in ecaps_report() can't be woken + * under the rq lock, so the last completion wakes it through irq_work. + */ +static void ecaps_report_done(struct scx_sched_pcpu *pcpu) +{ + pcpu->flags &= ~SCX_SCHED_PCPU_ECAPS_REPORT; + if (atomic_dec_and_test(&pcpu->sch->nr_ecaps_reports)) + irq_work_queue(&ecaps_report_irq_work); +} + /** * __scx_process_sync_ecaps - Sync this cpu's ecaps to pshard->caps[] * @rq: the cid's cpu rq * @prev: @rq's previous task from the in-progress dispatch * * pshard->caps[] is the target configuration. pcpu->ecaps is the effective - * transposed copy owned by the cid's cpu and written only here under @rq's - * lock. + * transposed copy owned by the cid's cpu and written under @rq's lock. * * A sched that newly gains baseline access here is owed an update_idle() so it * learns the cid's idle state. Such a gain arms the per-rq @@ -1113,6 +1152,7 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) struct scx_sched_pcpu *pcpu = container_of(pos, struct scx_sched_pcpu, ecaps_to_sync_node); struct scx_pshard *ps = cid_pshard(pcpu->sch, cid); + bool report = pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT; u64 old, ecaps, lost, gained; init_llist_node(pos); @@ -1120,8 +1160,17 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) /* pairs with smp_mb() in queue_sync_ecaps(), see there */ smp_mb(); + /* + * Before the sched's has_op is set, only the parent's op would + * run, and the change would never reach the sched. + * ecaps_enable() enables deliveries on each cpu once has_op is + * set. Until then, a sync leaves the ecaps alone. + */ old = READ_ONCE(pcpu->ecaps); - ecaps = calc_effective_caps(ps, cid); + if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED) + ecaps = calc_effective_caps(ps, cid); + else + ecaps = old; WRITE_ONCE(pcpu->ecaps, ecaps); lost = old & ~ecaps; @@ -1139,25 +1188,28 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) * up there. The rq recorded in it keeps the dispatches * targeting @rq. * - * Bypass propagates down the hierarchy, so a sched that isn't - * bypassing has no bypassing parent. The child's bypass state - * gates both deliveries. Both report the same before value, so - * one reported_ecaps covers them. + * Both report the same before value, so one reported_ecaps + * covers them. */ - if (ecaps != pcpu->reported_ecaps && !scx_bypassing(pcpu->sch, cpu)) { + if (ecaps != pcpu->reported_ecaps && + (pcpu->flags & SCX_SCHED_PCPU_ECAPS_ENABLED)) { struct scx_sched *parent = scx_parent(pcpu->sch); struct scx_dsp_ctx *dspc; /* stash @prev so nested dispatches can access it */ rq->scx.sub_dispatch_prev = prev; - if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated)) { + + /* a disabling sched's ops are not called */ + if (SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) && + likely(!READ_ONCE(pcpu->sch->aborting))) { dspc = &this_cpu_ptr(pcpu->sch->pcpu)->dsp_ctx; dspc->rq = rq; SCX_CALL_OP(pcpu->sch, sub_ecaps_updated, rq, scx_cpu_arg(cpu), pcpu->reported_ecaps, ecaps); scx_flush_dispatch_buf(pcpu->sch, rq); } - if (SCX_HAS_OP(parent, sub_child_ecaps_updated)) { + if (SCX_HAS_OP(parent, sub_child_ecaps_updated) && + likely(!READ_ONCE(parent->aborting))) { dspc = &this_cpu_ptr(parent->pcpu)->dsp_ctx; dspc->rq = rq; SCX_CALL_OP(parent, sub_child_ecaps_updated, rq, @@ -1169,6 +1221,9 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) pcpu->reported_ecaps = ecaps; } + if (report) + ecaps_report_done(pcpu); + /* * Gaining baseline access owes an update_idle() so the sched * learns the cpu's idle state. Arm the per-rq gate so the next @@ -1191,41 +1246,6 @@ void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) scx_schedule_reenq_local(rq, SCX_REENQ_CAP_REVOKE); } -/** - * scx_unbypass_replay_ecaps - Replay a bypass-suppressed ecaps notification - * @rq: rq of the cpu leaving bypass - * @sch: scheduler that just left bypass on @rq's cpu - * - * scx_process_sync_ecaps() consumes syncs while bypassing without delivering - * ops.sub_ecaps_updated(), leaving reported_ecaps stale. Nothing re-queues a - * sync when bypass lifts, so without a replay a cid that never changes again - * would never be notified. The attach-time initial grants are the acute case - * as they are consumed during the enable bypass window. Re-queue a sync for - * any undelivered delta so the next dispatch delivers it. - */ -void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) -{ - s32 cpu = cpu_of(rq); - struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); - struct scx_pshard *ps; - s32 cid; - - lockdep_assert_rq_held(rq); - - /* root holds every cap and never uses ecaps */ - if (!sch->level) - return; - - if (READ_ONCE(pcpu->ecaps) == pcpu->reported_ecaps) - return; - - cid = __scx_cpu_to_cid(cpu); - ps = sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]]; - - guard(raw_spinlock)(&ps->lock); - queue_sync_ecaps(sch, cid); -} - /* * A cpu came back. Re-seed each sub-sched's ecaps on the cpu's cid. The sync * recomputes effective caps from the pshard and fires ops.sub_ecaps_updated() @@ -1281,24 +1301,32 @@ void scx_offline_ecaps(struct rq *rq) root = scx_root_protected(); scx_for_each_descendant_pre(pos, root) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(pos->pcpu, cpu); + /* root holds every cap and never uses ecaps */ if (!pos->level) continue; - WRITE_ONCE(per_cpu_ptr(pos->pcpu, cpu)->ecaps, 0); + WRITE_ONCE(pcpu->ecaps, 0); + + /* deliveries have stopped on this cpu, clear its mark */ + if (pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT) + ecaps_report_done(pcpu); } } /* - * Clear every cap @sch holds. The pshard caps go first as they are the source a - * pending sync recomputes ecaps from. ecaps are then zeroed directly for the - * cap checks. + * Clear every cap @sch holds in the pshards. The report that follows delivers + * the loss and zeroes the ecaps. The caller holds off cpu hotplug across this + * and the report. */ static void clear_all_caps(struct scx_sched *sch) { s32 si, cpu; u32 cap_bit; + lockdep_assert_cpus_held(); + /* enable may have failed before scx_alloc_pshards() */ if (!sch->pshard) return; @@ -1312,8 +1340,13 @@ static void clear_all_caps(struct scx_sched *sch) } for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + guard(rq_lock_irqsave)(cpu_rq(cpu)); - WRITE_ONCE(per_cpu_ptr(sch->pcpu, cpu)->ecaps, 0); + + /* drop an unreported loss, as the report skips inactive cpus */ + if (!cpu_active(cpu)) + pcpu->reported_ecaps = 0; } } @@ -1331,6 +1364,9 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) guard(rq_lock_irqsave)(rq); + /* ecaps_report() waited for every mark it set */ + WARN_ON_ONCE(pcpu->flags & SCX_SCHED_PCPU_ECAPS_REPORT); + if (!llist_on_list(&pcpu->ecaps_to_sync_node)) return; @@ -1657,6 +1693,75 @@ static inline s32 scx_cgroup_claim_subtree(struct scx_sched *sch) { return 0; } static inline void scx_cgroup_return_subtree(struct scx_sched *sch) {} #endif +/* + * Enable @sch's ecaps deliveries on every possible cpu, so that a cpu inactive + * during the attach report delivers from the reseed at its activation. + */ +static void ecaps_enable(struct scx_sched *sch) +{ + s32 cpu; + + for_each_possible_cpu(cpu) { + guard(rq_lock_irqsave)(cpu_rq(cpu)); + per_cpu_ptr(sch->pcpu, cpu)->flags |= SCX_SCHED_PCPU_ECAPS_ENABLED; + } + sch->ecaps_enabled = true; +} + +/** + * ecaps_report - Report @sch's ecaps and wait + * @sch: sub-scheduler to report + * + * Mark every cpu @sch was ever granted a cap on, deliver its pending change + * through the dispatch path and wait for it. An inactive cpu is skipped. A cpu + * going down meanwhile has its mark cleared by scx_offline_ecaps(). + * + * Called with scx_enable_mutex held. A sched's attach and detach reports must + * not overlap, as the mark is one bit per pcpu. + */ +static void ecaps_report(struct scx_sched *sch) +{ + s32 cpu; + + lockdep_assert_held(&scx_enable_mutex); + + for_each_possible_cpu(cpu) { + struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu); + struct rq *rq = cpu_rq(cpu); + struct scx_pshard *ps; + s32 cid; + + guard(rq_lock_irqsave)(rq); + + /* syncs on an inactive cpu are discarded */ + if (!cpu_active(cpu)) + continue; + + cid = __scx_cpu_to_cid(cpu); + ps = cid_pshard(sch, cid); + guard(raw_spinlock)(&ps->lock); + + if (!pcpu->had_caps) + continue; + + pcpu->flags |= SCX_SCHED_PCPU_ECAPS_REPORT; + atomic_inc(&sch->nr_ecaps_reports); + queue_sync_ecaps(sch, cid); + + /* + * The report needs an ext dispatch on this cpu. A higher class + * task can hold the cpu, so also start the ext server, which + * picks the ext class within its period. + */ + update_rq_clock(rq); + dl_server_start(&rq->ext_server); + scx_resched_curr_deferred(rq); + } + + rcuwait_wait_event(&ecaps_report_wait, !atomic_read(&sch->nr_ecaps_reports), + TASK_UNINTERRUPTIBLE); +} + void scx_sub_disable(struct scx_sched *sch) { struct scx_sched *parent = scx_parent(sch); @@ -1678,6 +1783,24 @@ void scx_sub_disable(struct scx_sched *sch) * disable ops. */ mutex_lock(&scx_enable_mutex); + + /* + * Clear @sch's caps and report the loss before any task goes back to + * the parent. A disabling sched is granted nothing, so the caps stay + * cleared. Hold off cpu hotplug across both. The report covers every + * active cpu and the clear drops the unreported loss on the rest, so + * that no loss reaches the parent after ops.sub_detach(). + * + * An enable that failed before its attach report has nothing to report. + * If it failed before scx_link_sched(), the root's disable doesn't wait + * for it and may have retired the cid tables that the report reads. + */ + scoped_guard(cpus_read_lock) { + clear_all_caps(sch); + if (sch->ecaps_enabled) + ecaps_report(sch); + } + percpu_down_write(&scx_fork_rwsem); scx_cgroup_lock(); @@ -1785,12 +1908,6 @@ void scx_sub_disable(struct scx_sched *sch) scx_unlink_sched(sch); - /* - * The parent restores what it delegated in ops.sub_detach(). @sch must - * hold no caps by then. - */ - clear_all_caps(sch); - mutex_unlock(&scx_enable_mutex); /* @@ -1950,15 +2067,6 @@ void scx_sub_enable_workfn(struct kthread_work *work) if (ret) goto err_disable; - /* - * Bypass before @sch is linked and grants can reach it. The parent's - * delivery advances reported_ecaps, so a sync consumed before @sch's - * ops are registered would reach the parent and never be replayed to - * @sch. While bypassed, the syncs are consumed without a delivery and - * replayed to both at unbypass. - */ - scx_bypass(sch, true); - ret = scx_link_sched(sch); if (ret) goto err_disable; @@ -2004,6 +2112,8 @@ void scx_sub_enable_workfn(struct kthread_work *work) } sch->sub_attached = true; + scx_bypass(sch, true); + for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++) if (((void (**)(void))ops)[i]) set_bit(i, sch->has_op); @@ -2145,10 +2255,16 @@ void scx_sub_enable_workfn(struct kthread_work *work) scx_cgroup_unlock(); percpu_up_write(&scx_fork_rwsem); - scx_bypass(sch, false); - - /* @sch is enabled; deliver any caps owed since its sub_attach() */ + /* + * Report the caps granted from ops.sub_attach() before bypass lifts and + * @sch makes its first scheduling decision. The report sleeps until its + * syncs are processed, so it runs outside the fork rwsem. + */ scx_sub_seed_caps(sch); + ecaps_enable(sch); + ecaps_report(sch); + + scx_bypass(sch, false); pr_info("sched_ext: BPF sub-scheduler \"%s\" enabled\n", sch->ops.name); kobject_uevent(&sch->kobj, KOBJ_ADD); @@ -2578,8 +2694,12 @@ __bpf_kfunc s32 scx_bpf_sub_grant(u64 cgroup_id, u64 caps, * the cid gains baseline access, so the holder * learns of an already-idle cid. */ - scx_cmask_for_each_cid(cid, changed_cids) + scx_cmask_for_each_cid(cid, changed_cids) { + s32 cpu = __scx_cid_to_cpu(cid); + + per_cpu_ptr(child->pcpu, cpu)->had_caps = true; queue_sync_ecaps(child, cid); + } } } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index b76d750463f0..63769205c013 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -32,7 +32,6 @@ void scx_free_pshards(struct scx_sched *sch); s32 scx_alloc_pshards(struct scx_sched *sch); void scx_init_root_caps(struct scx_sched *sch); void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev); -void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch); void scx_online_ecaps(struct rq *rq); void scx_offline_ecaps(struct rq *rq); void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu); @@ -110,7 +109,6 @@ static inline void scx_free_pshards(struct scx_sched *sch) {} static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; } static inline void scx_init_root_caps(struct scx_sched *sch) {} static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {} -static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {} static inline void scx_online_ecaps(struct rq *rq) {} static inline void scx_offline_ecaps(struct rq *rq) {} static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {} diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 37d4808d7f19..22a2628a04a2 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -855,6 +855,7 @@ struct scx_rq { #endif struct irq_work deferred_irq_work; struct irq_work kick_cpus_irq_work; + struct irq_work resched_irq_work; }; #endif /* CONFIG_SCHED_CLASS_EXT */