diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index f1c833f1b78d..3645a844cfbc 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -388,8 +388,12 @@ static const struct sched_class *scx_setscheduler_class(struct task_struct *p) return __setscheduler_class(p->policy, p->prio); } -static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 cpu) +static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, struct rq *rq, + struct task_struct *p, u64 *enq_flags) { + struct scx_sched *host = scx_exec_sched(sch, cpu_of(rq)); + struct scx_dispatch_q *dsq; + /* * If @sch is a sub-sched which is bypassing, its tasks should go into * the bypass DSQs of the nearest ancestor which is not bypassing. The @@ -400,8 +404,29 @@ static struct scx_dispatch_q *bypass_enq_target_dsq(struct scx_sched *sch, s32 c * Whenever a sched starts bypassing, all runnable tasks in its subtree * are re-enqueued after scx_bypassing() is turned on, guaranteeing that * all tasks are transferred to the right DSQs. + * + * A cpu's dispatch only descends into a host holding %SCX_CAP_BASE on + * it, so the bypass DSQ of a host without BASE here is never consumed. + * Send @p to rescue on this cpu instead, resolving against @host. + * scx_dispatch_enqueue() resolves a local DSQ again, against the + * bypassing @sch. %SCX_ENQ_IGNORE_CAPS makes that second resolution + * admit @p. + * + * A migration-disabled task can't leave this cpu, and a task with a + * migration pending must run here until the migration moves it. Either + * would wait on the unconsumed bypass DSQ until the host regains BASE. + * Resolve them too when rescue is disabled. The resolution admits both + * to the local DSQ, as it does any insert of such a task. */ - return scx_bypass_dsq(scx_exec_sched(sch, cpu), cpu); + if (unlikely(scx_missing_caps(host, cpu_of(rq), SCX_CAP_BASE)) && + (scx_rescue_enabled() || is_migration_disabled(p) || p->migration_pending)) { + *enq_flags |= SCX_ENQ_RESCUE; + dsq = scx_resolve_local_dsq(host, rq, p, enq_flags); + *enq_flags |= SCX_ENQ_IGNORE_CAPS; + } else { + dsq = scx_bypass_dsq(host, cpu_of(rq)); + } + return dsq; } /** @@ -2353,7 +2378,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, dsq = find_global_dsq(sch, task_cpu(p)); goto enqueue; bypass: - dsq = bypass_enq_target_dsq(sch, task_cpu(p)); + dsq = bypass_enq_target_dsq(sch, rq, p, &enq_flags); goto enqueue; enqueue: @@ -3735,8 +3760,13 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * The exception is an IMMED donor put by proxy_resched_idle(): * the CPU isn't busy, proxy resolution is only retrying through * idle, so keep the donor where it can be picked again. + * + * A task with a kernel slice stays on the local DSQ even while + * @sch bypasses. Under bypass, rescue runs @p when the host + * lacks %SCX_CAP_BASE, see bypass_enq_target_dsq(). */ - if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) { + if (p->scx.slice && (!scx_bypassing(sch, cpu_of(rq)) || + scx_task_has_kernel_slice(p))) { if ((p->scx.flags & SCX_TASK_IMMED) && !proxy_put) { p->scx.flags |= SCX_TASK_REENQ_PREEMPTED; scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1); @@ -4987,11 +5017,14 @@ static void switched_from_scx(struct rq *rq, struct task_struct *p) * parent's own teardown). There is nothing to disable; calling * scx_disable_task() would WARN on the non-%ENABLED state and trigger a * NONE -> READY validation failure. + * + * @p can still be a rescuee under the parent. End its slice as + * scx_disable_task() does for an enabled task. */ if (scx_get_task_state(p) == SCX_TASK_NONE) - return; - - scx_disable_task(scx_task_sched(p), p); + scx_task_slice_ended(rq, p); + else + scx_disable_task(scx_task_sched(p), p); } static void switched_to_scx(struct rq *rq, struct task_struct *p) {} @@ -6930,7 +6963,8 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, * - ops.dispatch() is ignored. * * - ops.sub_ecaps_updated() and ops.sub_child_ecaps_updated() are still called. - * The bypassed tasks are on the bypass DSQs, out of their reach. + * The bypassed tasks are on the bypass DSQs, under rescue or admitted to a + * local DSQ, out of their reach. * * - dispatch_one() expires prev's slice before its keep test as the slice can't * be trusted. Each tick does the same and rotates the running task to the diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index ed63e29c9442..ecd677c4b561 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -924,8 +924,9 @@ struct sched_ext_ops { * ops.sub_attach() and any change since are reported then. A cpu that * isn't online at that point reports them once it comes online. * Deliveries continue while this sched bypasses. Its tasks are then on - * the bypass DSQs, where this op can't insert or move them. Deliveries - * stop once this sched is being disabled. + * the bypass DSQs, under rescue or admitted to a local DSQ, where this + * op can't insert or move them. Deliveries stop once this sched is + * being disabled. * * Until this sched is being disabled, whenever an op of this sched runs * under a cid's rq lock, its effective caps on the cid are the @after diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 6fa8cea1ef88..01215384eaa5 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -36,7 +36,7 @@ static DEFINE_STATIC_KEY_FALSE(__scx_ops_cid_sched_updated_enabled); static s32 scx_nr_ops_cid_sched_updated; /* such scheds, under scx_enable_mutex */ /* latched at root enable before any rescue runs */ -static s32 scx_rescue_bw_1024; +s32 scx_rescue_bw_1024; /* 0 if disabled, see scx_rescue_enabled() */ static s64 scx_rescue_quantum_ns; static s64 scx_rescue_sat_delta_ns; static unsigned long scx_rescue_decay_halflife; @@ -519,8 +519,8 @@ void scx_rescue_end(struct rq *rq) * rescuee. dispatch_one() keeps it running and put_prev_task_scx() reinserts it * at the tail of the local DSQ, behind whatever preempted the rescuee. * - * Return %false to end the rescue instead - the slice is already fully served, - * @p is leaving the rq or bypass is dismantling rescues. + * Return %false to end the rescue instead - the slice is already fully served + * or @p is leaving the rq. */ bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { @@ -528,8 +528,7 @@ bool scx_rescue_keep(struct rq *rq, struct task_struct *p) lockdep_assert_rq_held(rq); - if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED) || - scx_bypassing(scx_task_sched(p), cpu_of(rq))) + if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED)) return false; /* never called on a protected rescuee, so the write can't be refused */ @@ -887,9 +886,11 @@ void scx_rescue_init(struct rq *rq) * %SCX_ENQ_PREEMPT_LAZY are also cleared on a fallback migration-disabled * admission. * - * Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued - * to and run by its nearest non-bypassing ancestor. If root is bypassing, it - * always holds all caps. + * bypass_enq_target_dsq() also calls this with the host as @sch when the host + * lacks %SCX_CAP_BASE on @rq's cid, if rescue is enabled or @p must run on the + * cpu, migration-disabled or with a migration pending. The bypassed task is + * then admitted or diverted like a capless %SCX_ENQ_RESCUE insert by the host + * itself. */ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) @@ -932,7 +933,7 @@ struct scx_dispatch_q *__scx_resolve_local_dsq(struct scx_sched *sch, struct rq scx_divert_strip_flags(p, enq_flags); /* the enqueuer opted for rescue instead of rejection and reenqueue */ - if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) { + if (scx_rescue_enabled() && (*enq_flags & SCX_ENQ_RESCUE)) { __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1); if (scx_rescue_try_admit(rq, p)) return &rq->scx.local_dsq; @@ -1656,8 +1657,9 @@ static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) * * Used when @to must take over @p but failed to initialize it. Bypass keeps * scheduling decisions away from @to but @p can still trigger its task ops, - * which may confuse the BPF side. @to is dying anyway. The exit paths skip - * %NONE tasks (see __scx_disable_and_exit_task() and switched_from_scx()). + * which may confuse the BPF side. @to is dying anyway. The exit paths call no + * op on a %NONE task. __scx_disable_and_exit_task() skips it and + * switched_from_scx() only ends its slice. */ static void scx_punt_task(struct scx_sched *to, struct task_struct *p) { diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index de80a8af524c..b6d4a522a5a3 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -14,6 +14,8 @@ #ifdef CONFIG_EXT_SUB_SCHED +extern s32 scx_rescue_bw_1024; + struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root); struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root); void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch); @@ -51,6 +53,12 @@ void scx_rescue_dump(struct seq_buf *s, struct rq *rq); void scx_rescue_set_knobs(struct scx_sched *sch); void scx_rescue_init(struct rq *rq); +/* false only when the root disabled rescue, see scx_rescue_set_knobs() */ +static inline bool scx_rescue_enabled(void) +{ + return likely(scx_rescue_bw_1024); +} + /* * cgrp->scx_sched is written by root/sub enable/disable under all of * scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the @@ -128,6 +136,7 @@ static inline void scx_rescue_end(struct rq *rq) {} static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; } static inline void scx_rescue_flush(struct rq *rq) {} static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {} +static inline bool scx_rescue_enabled(void) { return false; } static inline void scx_rescue_set_knobs(struct scx_sched *sch) {} static inline void scx_rescue_init(struct rq *rq) {} static inline void scx_dec_has_subs(struct scx_sched *sch) {}