mirror of
https://github.com/torvalds/linux.git
synced 2026-11-06 21:58:01 +01:00
sched_ext: Fixes for v7.4
This is a big pull for fixes, but much of it was in the works before the
merge window. Most of these fixes are in sub-scheduler support, which nests
BPF schedulers under a parent that hands them CPUs. It was heavily reworked
over recent cycles and is now functionally complete. Porting real schedulers
on it drove the latest additions and keeps finding corner cases. For
example, once CPUs are shared and can be taken back, the kernel must keep
every task moving when its scheduler can't run it or is failing, and those
fallbacks interact subtly with each other and with other new core features.
- The fallback paths meant to keep a task moving when its scheduler can't
run it or is failing could themselves leave it stuck.
- The stall watchdog could blame a scheduler for time it wasn't responsible
for, even taking down all of sched_ext.
- Sub-schedulers were told late, or not at all, which CPUs they could use.
- Deferred callbacks could stay queued or be dropped under core scheduling
and proxy execution, with a sched/core fix to the duplicate check.
- Hardening: tracing programs can no longer call kfuncs that could deadlock
the CPU.
-----BEGIN PGP SIGNATURE-----
iIQEABYKACwWIQTfIjM1kS57o3GsC/uxYfJx3gVYGQUCauEJww4cdGpAa2VybmVs
Lm9yZwAKCRCxYfJx3gVYGah7AQD1rUZbNZrXHvV1cJ/WpB9UB6Ex05n4s5hDEbmZ
uMmCWgEAz3zfoly/Tbo4VvUPNVAyWj3gfpIwIPpwCklfODhIjAo=
=rDNB
-----END PGP SIGNATURE-----
Merge tag 'sched_ext-for-7.4-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext
Pull sched_ext fixes from Tejun Heo:
"This is a big pull for fixes, but much of it was in the works before
the merge window.
Most of these fixes are in sub-scheduler support, which nests BPF
schedulers under a parent that hands them CPUs. It was heavily
reworked over recent cycles and is now functionally complete. Porting
real schedulers on it drove the latest additions and keeps finding
corner cases.
For example, once CPUs are shared and can be taken back, the kernel
must keep every task moving when its scheduler can't run it or is
failing, and those fallbacks interact subtly with each other and with
other new core features.
- The fallback paths meant to keep a task moving when its scheduler
can't run it or is failing could themselves leave it stuck
- The stall watchdog could blame a scheduler for time it wasn't
responsible for, even taking down all of sched_ext
- Sub-schedulers were told late, or not at all, which CPUs they could
use
- Deferred callbacks could stay queued or be dropped under core
scheduling and proxy execution, with a sched/core fix to the
duplicate check
- Hardening: tracing programs can no longer call kfuncs that could
deadlock the CPU"
* tag 'sched_ext-for-7.4-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: (34 commits)
sched_ext: Reject scx_bpf_sub_grant/revoke() in tracing programs
sched_ext: Fix the rescue overload pick ejecting the wrong sub under bypass
sched_ext: Fix use-after-free of a NONE rescuee leaving its sched while running
sched_ext: Fix a lone IMMED task waiting behind a kept prev
sched_ext: Fix queued bypassed tasks stalling when the host loses SCX_CAP_BASE
sched_ext: Let the bypass load balancer move tasks homed on other cpus
sched_ext: Factor out bypass_lb_move() and insert with the donor DSQ unlocked
sched_ext: Fix the bypass load balancer stranding tasks on cpus where the host lacks SCX_CAP_BASE
sched_ext: Fix bypassed tasks stalling when the host lacks SCX_CAP_BASE
sched_ext: Expire prev's slice in dispatch_one() under bypass
sched_ext: Fix a cpu idling after put_prev_task_scx() queues its task on the local DSQ
sched_ext: Refuse extending a kernel slice
sched_ext: Mark the rescue grant as a kernel slice with SCX_TASK_KERNEL_SLICE
sched_ext: Fix a cpu idling with a runnable rescuee
sched_ext: Restore a rescuee's slice after ops.stopping()
sched_ext: Fix rescue served time disagreeing with the slice under proxy execution
sched_ext: Move refill_task_slice_dfl() below apply_slice_vtime()
sched_ext: Fix bypassed tasks looping on a host without SCX_CAP_ENQ
sched_ext: Fix bypassed tasks stranded by the reenqueue limit
sched_ext: Clear holding_cpu where the claim is confirmed
...
This commit is contained in:
Linus Torvalds
commit
729beb5aa8
|
|
@ -107,7 +107,21 @@ enum scx_ent_flags {
|
|||
SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */
|
||||
|
||||
/*
|
||||
* Bits 8 to 10 are used to carry task state:
|
||||
* The kernel owns the slice, currently the rescue grant. While set:
|
||||
*
|
||||
* - refill_task_slice_dfl() and expire_task_slice() leave the slice
|
||||
* alone.
|
||||
* - Reenqueue scans skip the task.
|
||||
* - put_prev_task_scx() keeps a preempted task on the local DSQ.
|
||||
* - Schedulers may shorten the slice, never extend it.
|
||||
*/
|
||||
SCX_TASK_KERNEL_SLICE = 1 << 8, /* slice owned by the kernel */
|
||||
|
||||
/*
|
||||
* The multi-bit fields are packed at the top, below SCX_TASK_CURSOR, so
|
||||
* that the single-bit flags stay contiguous from bit 0.
|
||||
*
|
||||
* Bits 25 to 27 are used to carry task state:
|
||||
*
|
||||
* NONE ops.init_task() not called yet
|
||||
* INIT_BEGIN ops.init_task() in flight; see sched_ext_dead()
|
||||
|
|
@ -116,7 +130,7 @@ enum scx_ent_flags {
|
|||
* ENABLED fully initialized and in sched_ext
|
||||
* DEAD terminal state set by sched_ext_dead()
|
||||
*/
|
||||
SCX_TASK_STATE_SHIFT = 8,
|
||||
SCX_TASK_STATE_SHIFT = 25,
|
||||
SCX_TASK_STATE_BITS = 3,
|
||||
SCX_TASK_STATE_MASK = ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT,
|
||||
|
||||
|
|
@ -128,7 +142,7 @@ enum scx_ent_flags {
|
|||
SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT,
|
||||
|
||||
/*
|
||||
* Bits 12 to 14 are used to carry reenqueue reason. In addition to
|
||||
* Bits 28 to 30 are used to carry reenqueue reason. In addition to
|
||||
* %SCX_ENQ_REENQ flag, ops.enqueue() can also test for
|
||||
* %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues.
|
||||
*
|
||||
|
|
@ -139,7 +153,7 @@ enum scx_ent_flags {
|
|||
* CAP sub-sched cap miss, see p->scx.reenq_reason_*
|
||||
* PROXY proxy state prevented a remote DSQ transfer
|
||||
*/
|
||||
SCX_TASK_REENQ_REASON_SHIFT = 12,
|
||||
SCX_TASK_REENQ_REASON_SHIFT = 28,
|
||||
SCX_TASK_REENQ_REASON_BITS = 3,
|
||||
SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT,
|
||||
|
||||
|
|
|
|||
|
|
@ -309,6 +309,12 @@ s32 scx_cid_init(struct scx_sched *sch)
|
|||
* empty shard.
|
||||
*/
|
||||
cids_in_core = cpumask_weight(core_scratch);
|
||||
if (cids_in_core > SCX_CID_SHARD_MAX_CPUS) {
|
||||
scx_error(sch, "core with %u cpus exceeds max shard size %d",
|
||||
cids_in_core, SCX_CID_SHARD_MAX_CPUS);
|
||||
return -E2BIG;
|
||||
}
|
||||
|
||||
max_cores = cores_per_shard + (shard_local < nr_large ? 1 : 0);
|
||||
if (cores_in_shard &&
|
||||
(cores_in_shard >= max_cores ||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -39,10 +39,18 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
|
|||
if (scx_consume_global_dsq(sch, rq))
|
||||
return SCX_DSP_LOCAL;
|
||||
|
||||
/*
|
||||
* A bypass consume puts a task on the cpu being dispatched for, to run
|
||||
* right away. That is an IMMED insert and needs only
|
||||
* %SCX_CAP_ENQ_IMMED, the base cap. Dispatch descends into @sch only on
|
||||
* a cpu where @sch holds the base cap, so @sch holds it here. The same
|
||||
* holds for the consume at the end of this function.
|
||||
*/
|
||||
if (scx_bypass_dsp_enabled(sch)) {
|
||||
/* if @sch is bypassing, only the bypass DSQs are active */
|
||||
if (scx_bypassing(sch, cpu)) {
|
||||
if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0))
|
||||
if (scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu),
|
||||
SCX_ENQ_IMMED))
|
||||
return SCX_DSP_LOCAL;
|
||||
return SCX_DSP_NONE;
|
||||
}
|
||||
|
|
@ -63,7 +71,8 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
|
|||
struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
|
||||
|
||||
if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
|
||||
scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
|
||||
scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu),
|
||||
SCX_ENQ_IMMED)) {
|
||||
__scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
|
||||
return SCX_DSP_LOCAL;
|
||||
}
|
||||
|
|
@ -135,7 +144,7 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
|
|||
* scheduler's ops.dispatch() doesn't yield any tasks.
|
||||
*/
|
||||
if (scx_bypass_dsp_enabled(sch) &&
|
||||
scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
|
||||
scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), SCX_ENQ_IMMED)) {
|
||||
__scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
|
||||
return SCX_DSP_LOCAL;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -878,9 +878,13 @@ struct sched_ext_ops {
|
|||
* @args: argument container, see the struct definition
|
||||
*
|
||||
* The sub-scheduler holds no caps by this point and can no longer
|
||||
* affect any cid. Its caps went away without a
|
||||
* sub_child_ecaps_updated() report, so whatever was delegated to it,
|
||||
* e.g. cpuperf targets, is the parent's to restore here.
|
||||
* affect any cid. Every cap reported through
|
||||
* ops.sub_child_ecaps_updated() was also reported lost there before the
|
||||
* sub-scheduler's tasks came back to this sched. Nothing is reported
|
||||
* for a sub-scheduler whose enable failed before its tasks were handed
|
||||
* over. If this sched is being disabled too, it can miss these reports.
|
||||
* On a cpu whose loss wasn't reported, whatever was delegated, e.g.
|
||||
* cpuperf targets, is the parent's to restore here.
|
||||
*/
|
||||
void (*sub_detach)(struct scx_sub_detach_args *args);
|
||||
|
||||
|
|
@ -914,6 +918,27 @@ struct sched_ext_ops {
|
|||
* change is in effect on the cpu. Runs in dispatch context with rq lock
|
||||
* held, and can perform all operations allowed in ops.dispatch()
|
||||
* including inserting/moving tasks.
|
||||
*
|
||||
* Deliveries start after this sched's tasks are handed to it and before
|
||||
* it makes its first scheduling decision. The caps granted from
|
||||
* ops.sub_attach() and any change since are reported then. A cpu that
|
||||
* isn't online at that point reports them once it comes online.
|
||||
* Deliveries continue while this sched bypasses. Its tasks are then on
|
||||
* the bypass DSQs, under rescue or admitted to a local DSQ, where this
|
||||
* op can't insert or move them. Deliveries stop once this sched is
|
||||
* being disabled.
|
||||
*
|
||||
* Until this sched is being disabled, whenever an op of this sched runs
|
||||
* under a cid's rq lock, its effective caps on the cid are the @after
|
||||
* of the last invocation for the cid.
|
||||
*
|
||||
* Deliveries for a cid happen only while the root considers the cid's
|
||||
* cpu online, never before the root's ops.cpu_online() or after its
|
||||
* ops.cpu_offline() for the cpu. A cpu going offline reports the loss
|
||||
* of every cap on it, and a cpu coming back reports the regain. A cpu
|
||||
* that suspend or hibernation takes down reports no loss and keeps its
|
||||
* caps while down. Only its net change, if any, is reported once it is
|
||||
* back.
|
||||
*/
|
||||
void (*sub_ecaps_updated)(s32 cid, u64 before, u64 after);
|
||||
|
||||
|
|
@ -940,11 +965,19 @@ struct sched_ext_ops {
|
|||
* @before: the child's effective caps as of the last delivery
|
||||
* @after: the child's effective caps now
|
||||
*
|
||||
* Invoked right after the child's sub_ecaps_updated(), once a grant or
|
||||
* revoke is in effect on the cpu. From here on the parent can act on a
|
||||
* Invoked right after the child's ops.sub_ecaps_updated(), once a
|
||||
* change is in effect on the cpu. From here on the parent can act on a
|
||||
* cap the child lost, e.g. reset the cpuperf target or schedule on the
|
||||
* cid. A detaching child's caps go away without a report, see
|
||||
* sub_detach(). A cpu going offline drops caps without a report.
|
||||
* cid. A disabling child gets no ops.sub_ecaps_updated(), so only this
|
||||
* op is invoked for its loss. A parent other than the root loses its
|
||||
* own caps on the cid too when the cpu goes offline, possibly before
|
||||
* this op is invoked, and regains them when the cpu comes back,
|
||||
* possibly after. It can't act on the cid while it holds no caps there.
|
||||
*
|
||||
* Deliveries start as for the child's ops.sub_ecaps_updated(), continue
|
||||
* while either sched bypasses and stop once this sched is being
|
||||
* disabled. A cpu going offline or coming back is reported as for the
|
||||
* child's ops.sub_ecaps_updated().
|
||||
*
|
||||
* Runs in dispatch context with rq lock held, and can perform all
|
||||
* operations allowed in ops.dispatch() including inserting/moving
|
||||
|
|
@ -962,6 +995,7 @@ struct sched_ext_ops {
|
|||
*
|
||||
* @cpu just came online. @cpu will not call ops.enqueue() or
|
||||
* ops.dispatch(), nor run tasks associated with other CPUs beforehand.
|
||||
* ops.sub_child_ecaps_updated() runs for @cpu only after this op.
|
||||
*/
|
||||
void (*cpu_online)(s32 cpu);
|
||||
|
||||
|
|
@ -971,6 +1005,7 @@ struct sched_ext_ops {
|
|||
*
|
||||
* @cpu is going offline. @cpu will not call ops.enqueue() or
|
||||
* ops.dispatch(), nor run tasks associated with other CPUs afterwards.
|
||||
* ops.sub_child_ecaps_updated() runs for @cpu only before this op.
|
||||
*/
|
||||
void (*cpu_offline)(s32 cpu);
|
||||
|
||||
|
|
@ -1309,8 +1344,10 @@ struct scx_event_stats {
|
|||
s64 SCX_EV_SLICE_CLAMPED;
|
||||
|
||||
/*
|
||||
* The number of times a slice extension was denied because the
|
||||
* scheduler lacked baseline cpu access on the task's cpu.
|
||||
* The number of times a slice write was refused. A raise is refused
|
||||
* without baseline cpu access on the task's cpu and on a kernel slice.
|
||||
* Every write is refused on a protected slice, a preemption's zeroing
|
||||
* included.
|
||||
*/
|
||||
s64 SCX_EV_SLICE_DENIED;
|
||||
|
||||
|
|
@ -1414,6 +1451,12 @@ struct scx_sched;
|
|||
|
||||
enum scx_sched_pcpu_flags {
|
||||
SCX_SCHED_PCPU_BYPASSING = 1LLU << 0,
|
||||
/* ecaps deliveries are enabled, see ecaps_enable() */
|
||||
SCX_SCHED_PCPU_ECAPS_ENABLED = 1LLU << 1,
|
||||
/* a report is pending, see ecaps_report() */
|
||||
SCX_SCHED_PCPU_ECAPS_REPORT = 1LLU << 2,
|
||||
/* a cpu going down waits for this sync, see scx_offline_ecaps() */
|
||||
SCX_SCHED_PCPU_ECAPS_OFFLINE = 1LLU << 3,
|
||||
};
|
||||
|
||||
/* dispatch buf */
|
||||
|
|
@ -1475,10 +1518,15 @@ struct scx_sched_pcpu {
|
|||
/* owed a forced update_idle() re-notify on this cpu */
|
||||
bool idle_renotify;
|
||||
/*
|
||||
* ecaps as of the last sub_ecaps_updated() and
|
||||
* sub_child_ecaps_updated() delivery. See __scx_process_sync_ecaps().
|
||||
* The sched was granted a cap on this cpu at some point. ecaps_report()
|
||||
* reschedules only these cpus, as rescheduling every cpu on each
|
||||
* sub-sched enable and disable would disrupt a large machine split many
|
||||
* ways. Exactly which cpus owe a delivery is hard to track, as
|
||||
* deliveries are asynchronous and reading caps[] and ecaps doesn't
|
||||
* tell. A cpu granted a cap at some point is a simple superset. Written
|
||||
* under pshard->lock.
|
||||
*/
|
||||
u64 reported_ecaps;
|
||||
bool had_caps;
|
||||
|
||||
/*
|
||||
* Decaying rescue runtime consumed on this cpu, see
|
||||
|
|
@ -1751,6 +1799,11 @@ struct scx_sched {
|
|||
|
||||
bool linked; /* on ->children, see scx_link_sched() */
|
||||
bool sub_attached;
|
||||
/* ecaps_enable() has run, under scx_enable_mutex */
|
||||
bool ecaps_enabled;
|
||||
|
||||
/* marked cpus whose sync is pending, see ecaps_report() */
|
||||
atomic_t nr_ecaps_reports;
|
||||
#endif /* CONFIG_EXT_SUB_SCHED */
|
||||
|
||||
/*
|
||||
|
|
@ -2107,12 +2160,13 @@ static const char *scx_enable_state_str[] = {
|
|||
* 4. Lock src_rq (where the task currently lives)
|
||||
* 5. Verify p->scx.holding_cpu == current CPU, if not, dequeue won the
|
||||
* race (dequeue clears holding_cpu to -1 when it takes the task), in
|
||||
* this case migration is aborted
|
||||
* 6. If src_rq == dst_rq: clear holding_cpu and enqueue directly
|
||||
* into dst_rq's local DSQ (no lock swap needed)
|
||||
* this case migration is aborted. Otherwise the claim is confirmed and
|
||||
* holding_cpu is cleared here, see unlink_dsq_and_switch_rq_lock()
|
||||
* 6. If src_rq == dst_rq: enqueue directly into dst_rq's local DSQ (no lock
|
||||
* swap needed)
|
||||
* 7. Otherwise, verify under src_rq lock that the task can be moved to dst_rq
|
||||
* (CPU affinity, migration_disabled, etc.). If not, clear holding_cpu,
|
||||
* leave the task on src_rq, and enqueue it on the fallback DSQ.
|
||||
* (CPU affinity, migration_disabled, etc.). If not, leave the task on
|
||||
* src_rq, and enqueue it on the fallback DSQ.
|
||||
* 8. Otherwise (i.e. if the task can be moved to dst_rq), call
|
||||
* move_remote_task_to_local_dsq(), which releases src_rq, locks dst_rq,
|
||||
* and performs the deactivate/activate migration cycle
|
||||
|
|
@ -2260,6 +2314,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq);
|
|||
s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch);
|
||||
__printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...);
|
||||
void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
|
||||
void scx_resched_curr_deferred(struct rq *rq);
|
||||
u64 __scx_bpf_now(struct rq *rq);
|
||||
void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
|
||||
u64 reenq_flags, struct rq *locked_rq);
|
||||
|
|
@ -2277,6 +2332,7 @@ void scx_cgroup_unlock(void);
|
|||
s32 scx_alloc_kern_arena_objs(struct scx_sched *sch);
|
||||
void scx_disable_bypass_dsp(struct scx_sched *sch);
|
||||
void scx_bypass(struct scx_sched *sch, bool bypass);
|
||||
void scx_bypass_lb_kick(struct scx_sched *sch);
|
||||
s32 scx_link_sched(struct scx_sched *sch);
|
||||
void scx_unlink_sched(struct scx_sched *sch);
|
||||
void scx_disable_dump(struct scx_sched *sch);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -14,6 +14,8 @@
|
|||
|
||||
#ifdef CONFIG_EXT_SUB_SCHED
|
||||
|
||||
extern s32 scx_rescue_bw_1024;
|
||||
|
||||
struct scx_sched *scx_skip_subtree_pre(struct scx_sched *pos, struct scx_sched *root);
|
||||
struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root);
|
||||
void scx_cid_sched_update(struct rq *rq, struct scx_sched *sch);
|
||||
|
|
@ -32,7 +34,10 @@ void scx_free_pshards(struct scx_sched *sch);
|
|||
s32 scx_alloc_pshards(struct scx_sched *sch);
|
||||
void scx_init_root_caps(struct scx_sched *sch);
|
||||
void __scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev);
|
||||
void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch);
|
||||
void scx_sub_pm_begin(void);
|
||||
void scx_update_frozen_cpus(s32 cpu, bool online);
|
||||
void scx_sub_pm_end(void);
|
||||
void scx_init_rq_ops_online(void);
|
||||
void scx_online_ecaps(struct rq *rq);
|
||||
void scx_offline_ecaps(struct rq *rq);
|
||||
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
|
||||
|
|
@ -48,6 +53,12 @@ void scx_rescue_dump(struct seq_buf *s, struct rq *rq);
|
|||
void scx_rescue_set_knobs(struct scx_sched *sch);
|
||||
void scx_rescue_init(struct rq *rq);
|
||||
|
||||
/* false only when the root disabled rescue, see scx_rescue_set_knobs() */
|
||||
static inline bool scx_rescue_enabled(void)
|
||||
{
|
||||
return likely(scx_rescue_bw_1024);
|
||||
}
|
||||
|
||||
/*
|
||||
* cgrp->scx_sched is written by root/sub enable/disable under all of
|
||||
* scx_enable_mutex, scx_fork_rwsem and cgroup_mutex. A new cgroup inherits the
|
||||
|
|
@ -110,7 +121,10 @@ static inline void scx_free_pshards(struct scx_sched *sch) {}
|
|||
static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
|
||||
static inline void scx_init_root_caps(struct scx_sched *sch) {}
|
||||
static inline void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) {}
|
||||
static inline void scx_unbypass_replay_ecaps(struct rq *rq, struct scx_sched *sch) {}
|
||||
static inline void scx_sub_pm_begin(void) {}
|
||||
static inline void scx_update_frozen_cpus(s32 cpu, bool online) {}
|
||||
static inline void scx_sub_pm_end(void) {}
|
||||
static inline void scx_init_rq_ops_online(void) {}
|
||||
static inline void scx_online_ecaps(struct rq *rq) {}
|
||||
static inline void scx_offline_ecaps(struct rq *rq) {}
|
||||
static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {}
|
||||
|
|
@ -122,6 +136,7 @@ static inline void scx_rescue_end(struct rq *rq) {}
|
|||
static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; }
|
||||
static inline void scx_rescue_flush(struct rq *rq) {}
|
||||
static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {}
|
||||
static inline bool scx_rescue_enabled(void) { return false; }
|
||||
static inline void scx_rescue_set_knobs(struct scx_sched *sch) {}
|
||||
static inline void scx_rescue_init(struct rq *rq) {}
|
||||
static inline void scx_dec_has_subs(struct scx_sched *sch) {}
|
||||
|
|
@ -248,12 +263,19 @@ static inline struct task_struct *scx_rescuee(struct rq *rq)
|
|||
return rq->scx.rescue.curr;
|
||||
}
|
||||
|
||||
/* does the kernel own @p's slice? see %SCX_TASK_KERNEL_SLICE */
|
||||
static inline bool scx_task_has_kernel_slice(const struct task_struct *p)
|
||||
{
|
||||
return scx_has_subs() && (p->scx.flags & SCX_TASK_KERNEL_SLICE);
|
||||
}
|
||||
|
||||
#else /* CONFIG_EXT_SUB_SCHED */
|
||||
|
||||
static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; }
|
||||
static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; }
|
||||
static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; }
|
||||
static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; }
|
||||
static inline bool scx_task_has_kernel_slice(const struct task_struct *p) { return false; }
|
||||
|
||||
#endif /* CONFIG_EXT_SUB_SCHED */
|
||||
|
||||
|
|
|
|||
|
|
@ -781,12 +781,21 @@ struct cfs_rq {
|
|||
/* scx_rq->flags, protected by the rq lock */
|
||||
enum scx_rq_flags {
|
||||
/*
|
||||
* A hotplugged CPU starts scheduling before rq_online_scx(). Track
|
||||
* ops.cpu_on/offline() state so that ops.enqueue/dispatch() are called
|
||||
* only while the BPF scheduler considers the CPU to be online.
|
||||
* Set while the core has the rq online, see rq_online_scx() and
|
||||
* rq_offline_scx(). A hotplugged cpu starts scheduling before
|
||||
* rq_online_scx(). ops.enqueue() and ops.dispatch() are called for the
|
||||
* cpu only while it is set, see scx_rq_online() and
|
||||
* scx_dispatch_sched().
|
||||
*/
|
||||
SCX_RQ_ONLINE = 1 << 0,
|
||||
SCX_RQ_CAN_STOP_TICK = 1 << 1,
|
||||
/*
|
||||
* Set while the root considers the cpu online, from its
|
||||
* ops.cpu_online() until its ops.cpu_offline(). ecaps are delivered
|
||||
* only while it is set. %SCX_RQ_ONLINE follows the core's rq state
|
||||
* instead, which changes ahead of the root's view on both edges.
|
||||
*/
|
||||
SCX_RQ_OPS_ONLINE = 1 << 2,
|
||||
SCX_RQ_CLK_VALID = 1 << 5, /* RQ clock is fresh and valid */
|
||||
SCX_RQ_BAL_CB_PENDING = 1 << 6, /* must queue a cb after dispatching */
|
||||
SCX_RQ_SUB_IDLE_RENOTIFY = 1 << 7, /* sub-scheds are owed update_idle() */
|
||||
|
|
@ -807,7 +816,7 @@ struct scx_rq_rescue {
|
|||
u64 clock; /* last budget accrual timestamp */
|
||||
struct task_struct *curr; /* task being rescued, one at a time */
|
||||
s64 slice; /* curr's admitted slice */
|
||||
u64 exec_snap; /* sum_exec_runtime at admission */
|
||||
s64 served; /* curr's served time */
|
||||
struct timer_list timer; /* paces admission and escalation */
|
||||
u64 kill_at; /* last ejection, init before any */
|
||||
};
|
||||
|
|
@ -838,9 +847,10 @@ struct scx_rq {
|
|||
struct scx_sched *sched;
|
||||
struct llist_head ecaps_to_sync; /* pending ecaps syncs */
|
||||
struct task_struct *sub_dispatch_prev;
|
||||
u32 nr_ecaps_offline; /* marked syncs a down waits for */
|
||||
#endif
|
||||
cpumask_var_t cpus_to_sync;
|
||||
bool kick_sync_pending;
|
||||
cpumask_var_t cpus_to_sync;
|
||||
unsigned long kick_sync;
|
||||
|
||||
struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */
|
||||
|
|
@ -855,6 +865,7 @@ struct scx_rq {
|
|||
#endif
|
||||
struct irq_work deferred_irq_work;
|
||||
struct irq_work kick_cpus_irq_work;
|
||||
struct irq_work resched_irq_work;
|
||||
};
|
||||
#endif /* CONFIG_SCHED_CLASS_EXT */
|
||||
|
||||
|
|
@ -2167,7 +2178,8 @@ queue_balance_callback(struct rq *rq,
|
|||
* balance_push() is active, see the comment with
|
||||
* balance_push_callback.
|
||||
*/
|
||||
if (unlikely(head->next || rq->balance_callback == &balance_push_callback))
|
||||
if (unlikely(head->next || rq->balance_callback == head ||
|
||||
rq->balance_callback == &balance_push_callback))
|
||||
return;
|
||||
|
||||
head->func = func;
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@
|
|||
#define HAVE_SCX_TASK_IMMED
|
||||
#define HAVE_SCX_TASK_PROTECTED
|
||||
#define HAVE_SCX_TASK_RUN_TRACKED
|
||||
#define HAVE_SCX_TASK_KERNEL_SLICE
|
||||
#define HAVE_SCX_TASK_STATE_SHIFT
|
||||
#define HAVE_SCX_TASK_STATE_BITS
|
||||
#define HAVE_SCX_TASK_STATE_MASK
|
||||
|
|
|
|||
|
|
@ -120,24 +120,25 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[]
|
|||
{ "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_KERNEL_SLICE", 0x100LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x19LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_STATE_MASK", 0xe000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_NONE", 0x0LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_INIT_BEGIN", 0x100LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_INIT", 0x200LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_READY", 0x300LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_ENABLED", 0x400LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_DEAD", 0x500LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0xcLLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_INIT_BEGIN", 0x2000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_INIT", 0x4000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_READY", 0x6000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_ENABLED", 0x8000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_DEAD", 0xa000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0x1cLLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_REASON_BITS", 0x3LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x7000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x70000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_NONE", 0x0LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x1000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x5000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x10000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x20000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x30000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x40000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_REENQ_PROXY", 0x50000000LLU },
|
||||
{ "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU },
|
||||
{ "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU },
|
||||
{ "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU },
|
||||
|
|
|
|||
|
|
@ -2180,35 +2180,16 @@ s32 BPF_STRUCT_OPS(qmap_sub_attach, struct scx_sub_attach_args *args)
|
|||
void BPF_STRUCT_OPS(qmap_sub_detach, struct scx_sub_detach_args *args)
|
||||
{
|
||||
u64 cgid = args->ops->sub_cgroup_id;
|
||||
s32 nr_cids = qa.nr_cids;
|
||||
s32 pos = qa.part.rr_pos;
|
||||
u64 holder_cgid;
|
||||
bool reset_pool;
|
||||
s32 i, cid;
|
||||
|
||||
if (nr_cids < 0 || nr_cids > SCX_QMAP_MAX_CPUS || pos < 0 || pos >= MAX_PARTS) {
|
||||
scx_bpf_error("-ERANGE");
|
||||
return;
|
||||
}
|
||||
s32 i;
|
||||
|
||||
/*
|
||||
* The child's caps are gone without a report, see ops.sub_detach().
|
||||
* cpuperf targets persist until the next write. Reset the child's excl
|
||||
* cids, and the pool unless another child holds it: that child's revoke
|
||||
* report or its own detach resets it then.
|
||||
* qmap_sub_child_ecaps_updated() has reset the cpuperf targets on the
|
||||
* loss reports.
|
||||
*/
|
||||
holder_cgid = qa.part.rr_slots[pos];
|
||||
reset_pool = !holder_cgid || holder_cgid == cgid;
|
||||
|
||||
for (i = 0; i < MAX_SUB_SCHEDS; i++) {
|
||||
if (qa.sub_sched_ctxs[i].cgroup_id != cgid)
|
||||
continue;
|
||||
|
||||
bpf_arena_for(cid, 0, nr_cids)
|
||||
if (cmask_test(cid, &qa.sub_sched_ctxs[i].granted_cids.mask) ||
|
||||
(reset_pool && cmask_test(cid, &qa.rr_cids.mask)))
|
||||
scx_bpf_cidperf_set(cid, SCX_CPUPERF_ONE);
|
||||
|
||||
qa.sub_sched_ctxs[i].cgroup_id = 0;
|
||||
qa.sub_sched_ctxs[i].weight = 100;
|
||||
cmask_init(&qa.sub_sched_ctxs[i].granted_cids.mask, 0, qa.nr_cids);
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user