diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index b341c328948e..fcf85b87fdd4 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -106,6 +106,17 @@ enum scx_ent_flags { SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */ + /* + * The kernel owns the slice, currently the rescue grant. While set: + * + * - refill_task_slice_dfl() and expire_task_slice() leave the slice + * alone. + * - Reenqueue scans skip the task. + * - put_prev_task_scx() keeps a preempted task on the local DSQ. + * - Scheduler slice writes and preemption still apply. + */ + SCX_TASK_KERNEL_SLICE = 1 << 8, /* slice owned by the kernel */ + /* * The multi-bit fields are packed at the top, below SCX_TASK_CURSOR, so * that the single-bit flags stay contiguous from bit 0. diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index 35e2fef66c04..01b0fbf51722 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1472,8 +1472,8 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice) * @rq: rq @p is on * @p: task of interest * - * End what rides on the slice - the protection, and the rescue if @p is being - * rescued. + * End what rides on the slice - the protection, the kernel's ownership of the + * slice, and the rescue if @p is being rescued. * * A dequeue normally ends the slice too. The exception is a save/restore pair * on the running task. Attribute changes like renice cycle the task through @@ -1485,7 +1485,7 @@ void scx_task_slice_ended(struct rq *rq, struct task_struct *p) { lockdep_assert_rq_held(rq); - p->scx.flags &= ~SCX_TASK_PROTECTED; + p->scx.flags &= ~(SCX_TASK_PROTECTED | SCX_TASK_KERNEL_SLICE); if (unlikely(p == scx_rescuee(rq))) scx_rescue_end(rq); } @@ -1569,8 +1569,16 @@ static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 e p->scx.dsq_vtime = vtime; } +/* + * refill_task_slice_dfl() supplies a slice when the scheduler left none and + * expire_task_slice() takes it away to force the task off the cpu. Neither + * touches a kernel slice, see %SCX_TASK_KERNEL_SLICE. + */ static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) { + if (scx_task_has_kernel_slice(p)) + return; + /* * A default refill is not an explicit request, so it must not drop a * pending out-of-band one, which is applied when @p next runs. @@ -1579,6 +1587,17 @@ static void refill_task_slice_dfl(struct scx_sched *sch, struct task_struct *p) __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1); } +static void expire_task_slice(struct task_struct *p) +{ + /* + * Unlike the refill above, an expiry supersedes a pending out-of-band + * request. A kept request would be applied at the next + * update_curr_scx() and undo the expiry. + */ + if (!scx_task_has_kernel_slice(p)) + scx_set_task_slice(p, 0); +} + static void update_curr_scx(struct rq *rq) { struct task_struct *donor; @@ -3706,11 +3725,15 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, u64 enq_flags = 0; /* - * Keep a preempted rescue going. If preempted - * by another SCX task, append to the local DSQ, - * see scx_rescue_keep(). + * The enqueue caps gate the scheduler's + * placement. The kernel placed a kernel-slice + * task here itself, so reinsert it without a + * cap check, at the head like any preempted + * task. When scx_rescue_keep() restored a slice + * the scheduler zeroed, the scheduler wanted + * something else to run first, so append. */ - if (unlikely(p == scx_rescuee(rq))) { + if (unlikely(scx_task_has_kernel_slice(p))) { enq_flags |= SCX_ENQ_IGNORE_CAPS; if (!rescue_keep) enq_flags |= SCX_ENQ_HEAD; @@ -4417,7 +4440,7 @@ static void task_tick_scx(struct rq *rq, struct task_struct *donor, int queued) * management. */ if (scx_bypassing(sch, cpu_of(rq))) - scx_set_task_slice(donor, 0); + expire_task_slice(donor); else if (SCX_HAS_OP(sch, tick)) SCX_CALL_OP_TASK(sch, tick, rq, donor); @@ -5040,7 +5063,7 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p, first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST); *reenq_flags |= SCX_REENQ_TSR_NOT_FIRST; - if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq))) + if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || scx_task_has_kernel_slice(p))) return false; *reason = SCX_TASK_REENQ_KFUNC; @@ -5143,7 +5166,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags) if ((reenq_flags & SCX_REENQ_CAP_REVOKE) && rq->donor->sched_class == &ext_sched_class && scx_task_reenq_on_cap_revoke(rq, rq->donor)) { - scx_set_task_slice(rq->donor, 0); + expire_task_slice(rq->donor); resched_curr(rq); } @@ -6999,7 +7022,7 @@ void scx_bypass(struct scx_sched *sch, bool bypass) if (bypass) scx_task_slice_ended(rq, p); else if (!scx_task_can_stay_on_cpu(rq, p)) - scx_set_task_slice(p, 0); + expire_task_slice(p); } /* diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c index 5fa9c660aaa6..139257f3411f 100644 --- a/kernel/sched/ext/sub.c +++ b/kernel/sched/ext/sub.c @@ -616,10 +616,11 @@ static void scx_rescue_timer_arm(struct rq *rq) * @p: task being admitted, off any DSQ * @slice: CPU time to grant * - * The schedulers keep their normal control over @p and may preempt or reslice - * it. @slice is measured on served CPU time, so neither shortens the rescue, - * see scx_rescue_charge() and scx_rescue_keep(). Prolonged denial escalates - * into protected execution, see scx_rescue_timerfn(). + * The slice is a kernel slice, see %SCX_TASK_KERNEL_SLICE. The schedulers keep + * their normal control over @p and may preempt or reslice it. @slice is + * measured on served CPU time, so neither shortens the rescue, see + * scx_rescue_charge() and scx_rescue_keep(). Prolonged denial escalates into + * protected execution, see scx_rescue_timerfn(). */ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) { @@ -630,6 +631,7 @@ static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice) rq->scx.rescue.slice = slice; rq->scx.rescue.served = 0; scx_set_task_slice(p, slice); + p->scx.flags |= SCX_TASK_KERNEL_SLICE; scx_rescue_timer_arm(rq); } diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h index eb1dba6d51e2..de80a8af524c 100644 --- a/kernel/sched/ext/sub.h +++ b/kernel/sched/ext/sub.h @@ -254,12 +254,19 @@ static inline struct task_struct *scx_rescuee(struct rq *rq) return rq->scx.rescue.curr; } +/* does the kernel own @p's slice? see %SCX_TASK_KERNEL_SLICE */ +static inline bool scx_task_has_kernel_slice(const struct task_struct *p) +{ + return scx_has_subs() && (p->scx.flags & SCX_TASK_KERNEL_SLICE); +} + #else /* CONFIG_EXT_SUB_SCHED */ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; } static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; } static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; } static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; } +static inline bool scx_task_has_kernel_slice(const struct task_struct *p) { return false; } #endif /* CONFIG_EXT_SUB_SCHED */ diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 4fa82033b99c..362e8bb34499 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -108,6 +108,7 @@ #define HAVE_SCX_TASK_IMMED #define HAVE_SCX_TASK_PROTECTED #define HAVE_SCX_TASK_RUN_TRACKED +#define HAVE_SCX_TASK_KERNEL_SLICE #define HAVE_SCX_TASK_STATE_SHIFT #define HAVE_SCX_TASK_STATE_BITS #define HAVE_SCX_TASK_STATE_MASK diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h index 05a460611c4d..ad8f56f581ad 100644 --- a/tools/sched_ext/include/scx/enums_abi.autogen.h +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -120,6 +120,7 @@ static const struct __scx_enum_abi_val __scx_enum_abi_vals[] { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, { "scx_ent_flags", "SCX_TASK_RUN_TRACKED", 0x80LLU }, + { "scx_ent_flags", "SCX_TASK_KERNEL_SLICE", 0x100LLU }, { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x19LLU }, { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0xe000000LLU },